From 5241c9f3cfefd8da67c2e07de1864ac956f9788d Mon Sep 17 00:00:00 2001 From: Luca Deri Date: Fri, 21 Dec 2018 18:25:44 +0100 Subject: Added Modbus over TCP dissector --- src/lib/ndpi_main.c | 17 ++++++----- src/lib/protocols/modbus.c | 72 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 82 insertions(+), 7 deletions(-) create mode 100644 src/lib/protocols/modbus.c (limited to 'src/lib') diff --git a/src/lib/ndpi_main.c b/src/lib/ndpi_main.c index e882feaa6..b8d8f358d 100644 --- a/src/lib/ndpi_main.c +++ b/src/lib/ndpi_main.c @@ -1238,11 +1238,11 @@ static void ndpi_init_protocol_defaults(struct ndpi_detection_module_struct *ndp no_master, "NestLogSink", NDPI_PROTOCOL_CATEGORY_CLOUD, ndpi_build_default_ports(ports_a, 11095, 0, 0, 0, 0) /* TCP */, ndpi_build_default_ports(ports_b, 0, 0, 0, 0, 0) /* UDP */); - ndpi_set_proto_defaults(ndpi_mod, NDPI_PROTOCOL_FUN, NDPI_PROTOCOL_FREE_44, - 0 /* can_have_a_subprotocol */, no_master, - no_master, "Free", NDPI_PROTOCOL_CATEGORY_CUSTOM_1 /* dummy */, - ndpi_build_default_ports(ports_a, 0, 0, 0, 0, 0) /* TCP */, - ndpi_build_default_ports(ports_b, 0, 0, 0, 0, 0) /* UDP */); + ndpi_set_proto_defaults(ndpi_mod, NDPI_PROTOCOL_ACCEPTABLE, NDPI_PROTOCOL_MODBUS, + 1 /* no subprotocol */, no_master, + no_master, "Modbus", NDPI_PROTOCOL_CATEGORY_NETWORK, /* Perhaps IoT in the future */ + ndpi_build_default_ports(ports_a, 502, 0, 0, 0, 0) /* TCP */, + ndpi_build_default_ports(ports_b, 0, 0, 0, 0, 0) /* UDP */); ndpi_set_proto_defaults(ndpi_mod, NDPI_PROTOCOL_FUN, NDPI_PROTOCOL_FREE_45, 0 /* can_have_a_subprotocol */, no_master, no_master, "Free", NDPI_PROTOCOL_CATEGORY_CUSTOM_1 /* dummy */, @@ -1699,7 +1699,7 @@ static void ndpi_init_protocol_defaults(struct ndpi_detection_module_struct *ndp 0 /* can_have_a_subprotocol */, no_master, no_master, "UPnP", NDPI_PROTOCOL_CATEGORY_NETWORK, ndpi_build_default_ports(ports_a, 1780, 0, 0, 0, 0) /* TCP */, - ndpi_build_default_ports(ports_b, 1900, 0, 0, 0, 0) /* UDP */); /* Missing dissector: port based only */ + ndpi_build_default_ports(ports_b, 1900, 0, 0, 0, 0) /* UDP */); ndpi_set_proto_defaults(ndpi_mod, NDPI_PROTOCOL_ACCEPTABLE, NDPI_PROTOCOL_TELEGRAM, 0 /* can_have_a_subprotocol */, no_master, no_master, "Telegram", NDPI_PROTOCOL_CATEGORY_CHAT, @@ -3285,6 +3285,9 @@ void ndpi_set_protocol_detection_bitmask2(struct ndpi_detection_module_struct *n /* NINTENDO */ init_nintendo_dissector(ndpi_struct, &a, detection_bitmask); + /* MODBUS */ + init_modbus_dissector(ndpi_struct, &a, detection_bitmask); + /*** Put false-positive sensitive protocols at the end ***/ /* SKYPE */ @@ -4325,7 +4328,7 @@ int ndpi_fill_ip_protocol_category(struct ndpi_detection_module_struct *ndpi_str if(ndpi_struct->custom_categories.categories_loaded) { prefix_t prefix; patricia_node_t *node; - + /* Make sure all in network byte order otherwise compares wont work */ fill_prefix_v4(&prefix, (struct in_addr *)&iph->saddr, 32, ((patricia_tree_t*)ndpi_struct->protocols_ptree)->maxbits); diff --git a/src/lib/protocols/modbus.c b/src/lib/protocols/modbus.c new file mode 100644 index 000000000..2a6dd2a49 --- /dev/null +++ b/src/lib/protocols/modbus.c @@ -0,0 +1,72 @@ + +/* + * modbus.c + * + * Copyright (C) 2018 - ntop.org + * + * This file is part of nDPI, an open source deep packet inspection + * library based on the OpenDPI and PACE technology by ipoque GmbH + * + * nDPI is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * nDPI is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with nDPI. If not, see . + * + */ + + +#include "ndpi_protocol_ids.h" +#include "ndpi_api.h" + +#define NDPI_CURRENT_PROTO NDPI_PROTOCOL_MODBUS + +void ndpi_search_modbus_tcp(struct ndpi_detection_module_struct *ndpi_struct, + struct ndpi_flow_struct *flow) { + struct ndpi_packet_struct *packet = &flow->packet; + NDPI_LOG_DBG(ndpi_struct, "search Modbus\n"); + u_int16_t modbus_port = htons(502); // port used by modbus + + /* Check connection over TCP */ + + if(packet->tcp) { + /* The payload of Modbus-TCP segment must be at least 8 bytes (7 bytes of header application + packet plus 1 byte of minimum payload of application packet) + */ + if((packet->payload_packet_len >= 8) + &&((packet->tcp->dest == modbus_port) || (packet->tcp->source == modbus_port))) { + // Modbus uses the port 502 + u_int16_t modbus_len = htons(*((u_int16_t*)&packet->payload[4])); + + // the fourth parameter of the payload is the length of the segment + if((modbus_len-1) == (packet->payload_packet_len - 7 /* ModbusTCP header len */)) { + NDPI_LOG_INFO(ndpi_struct, "found MODBUS\n"); + ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_MODBUS, NDPI_PROTOCOL_UNKNOWN); + return; + } + } + } + NDPI_EXCLUDE_PROTO(ndpi_struct, flow); + +} + + + +void init_modbus_dissector(struct ndpi_detection_module_struct *ndpi_struct, + u_int32_t *id, NDPI_PROTOCOL_BITMASK *detection_bitmask) { + + ndpi_set_bitmask_protocol_detection("Modbus", ndpi_struct, detection_bitmask, *id, + NDPI_PROTOCOL_MODBUS, + ndpi_search_modbus_tcp, + NDPI_SELECTION_BITMASK_PROTOCOL_V4_V6_TCP_WITH_PAYLOAD_WITHOUT_RETRANSMISSION, + SAVE_DETECTION_BITMASK_AS_UNKNOWN, + ADD_TO_DETECTION_BITMASK); + *id += 1; +} -- cgit v1.2.3 From 97bdfe295d3f7318d1eac3e0020b3b13004f008d Mon Sep 17 00:00:00 2001 From: Luca Deri Date: Mon, 7 Jan 2019 00:28:29 +0100 Subject: nDPi now finally honours dissection of HTTP responses --- src/include/ndpi_typedefs.h | 443 ++++++++++++++++++++--------------------- src/lib/ndpi_main.c | 43 ++-- src/lib/protocols/apple_push.c | 30 +-- src/lib/protocols/http.c | 263 +++++++++++++----------- 4 files changed, 395 insertions(+), 384 deletions(-) (limited to 'src/lib') diff --git a/src/include/ndpi_typedefs.h b/src/include/ndpi_typedefs.h index 294af22b3..34d308e2e 100644 --- a/src/include/ndpi_typedefs.h +++ b/src/include/ndpi_typedefs.h @@ -29,18 +29,18 @@ /* NDPI_LOG_LEVEL */ typedef enum { - NDPI_LOG_ERROR, - NDPI_LOG_TRACE, - NDPI_LOG_DEBUG, - NDPI_LOG_DEBUG_EXTRA + NDPI_LOG_ERROR, + NDPI_LOG_TRACE, + NDPI_LOG_DEBUG, + NDPI_LOG_DEBUG_EXTRA } ndpi_log_level_t; /* NDPI_VISIT */ typedef enum { - ndpi_preorder, - ndpi_postorder, - ndpi_endorder, - ndpi_leaf + ndpi_preorder, + ndpi_postorder, + ndpi_endorder, + ndpi_leaf } ndpi_VISIT; /* NDPI_NODE */ @@ -356,15 +356,15 @@ PACK_ON struct tinc_cache_entry { } PACK_OFF; typedef enum { - HTTP_METHOD_UNKNOWN = 0, - HTTP_METHOD_OPTIONS, - HTTP_METHOD_GET, - HTTP_METHOD_HEAD, - HTTP_METHOD_POST, - HTTP_METHOD_PUT, - HTTP_METHOD_DELETE, - HTTP_METHOD_TRACE, - HTTP_METHOD_CONNECT + HTTP_METHOD_UNKNOWN = 0, + HTTP_METHOD_OPTIONS, + HTTP_METHOD_GET, + HTTP_METHOD_HEAD, + HTTP_METHOD_POST, + HTTP_METHOD_PUT, + HTTP_METHOD_DELETE, + HTTP_METHOD_TRACE, + HTTP_METHOD_CONNECT } ndpi_http_method; struct ndpi_id_struct { @@ -375,229 +375,229 @@ struct ndpi_id_struct { to compare this, use: **/ NDPI_PROTOCOL_BITMASK detected_protocol_bitmask; -/* NDPI_PROTOCOL_RTSP */ + /* NDPI_PROTOCOL_RTSP */ ndpi_ip_addr_t rtsp_ip_address; -/* NDPI_PROTOCOL_YAHOO */ + /* NDPI_PROTOCOL_YAHOO */ u_int32_t yahoo_video_lan_timer; /* NDPI_PROTOCOL_IRC_MAXPORT % 2 must be 0 */ -/* NDPI_PROTOCOL_IRC */ + /* NDPI_PROTOCOL_IRC */ #define NDPI_PROTOCOL_IRC_MAXPORT 8 u_int16_t irc_port[NDPI_PROTOCOL_IRC_MAXPORT]; u_int32_t last_time_port_used[NDPI_PROTOCOL_IRC_MAXPORT]; u_int32_t irc_ts; -/* NDPI_PROTOCOL_GNUTELLA */ + /* NDPI_PROTOCOL_GNUTELLA */ u_int32_t gnutella_ts; -/* NDPI_PROTOCOL_BATTLEFIELD */ + /* NDPI_PROTOCOL_BATTLEFIELD */ u_int32_t battlefield_ts; -/* NDPI_PROTOCOL_THUNDER */ + /* NDPI_PROTOCOL_THUNDER */ u_int32_t thunder_ts; -/* NDPI_PROTOCOL_RTSP */ + /* NDPI_PROTOCOL_RTSP */ u_int32_t rtsp_timer; -/* NDPI_PROTOCOL_OSCAR */ + /* NDPI_PROTOCOL_OSCAR */ u_int32_t oscar_last_safe_access_time; -/* NDPI_PROTOCOL_ZATTOO */ + /* NDPI_PROTOCOL_ZATTOO */ u_int32_t zattoo_ts; -/* NDPI_PROTOCOL_UNENCRYPTED_JABBER */ + /* NDPI_PROTOCOL_UNENCRYPTED_JABBER */ u_int32_t jabber_stun_or_ft_ts; -/* NDPI_PROTOCOL_DIRECTCONNECT */ + /* NDPI_PROTOCOL_DIRECTCONNECT */ u_int32_t directconnect_last_safe_access_time; -/* NDPI_PROTOCOL_SOULSEEK */ + /* NDPI_PROTOCOL_SOULSEEK */ u_int32_t soulseek_last_safe_access_time; -/* NDPI_PROTOCOL_DIRECTCONNECT */ + /* NDPI_PROTOCOL_DIRECTCONNECT */ u_int16_t detected_directconnect_port; u_int16_t detected_directconnect_udp_port; u_int16_t detected_directconnect_ssl_port; -/* NDPI_PROTOCOL_BITTORRENT */ + /* NDPI_PROTOCOL_BITTORRENT */ #define NDPI_BT_PORTS 8 u_int16_t bt_port_t[NDPI_BT_PORTS]; u_int16_t bt_port_u[NDPI_BT_PORTS]; -/* NDPI_PROTOCOL_UNENCRYPTED_JABBER */ + /* NDPI_PROTOCOL_UNENCRYPTED_JABBER */ #define JABBER_MAX_STUN_PORTS 6 u_int16_t jabber_voice_stun_port[JABBER_MAX_STUN_PORTS]; u_int16_t jabber_file_transfer_port[2]; -/* NDPI_PROTOCOL_GNUTELLA */ + /* NDPI_PROTOCOL_GNUTELLA */ u_int16_t detected_gnutella_port; -/* NDPI_PROTOCOL_GNUTELLA */ + /* NDPI_PROTOCOL_GNUTELLA */ u_int16_t detected_gnutella_udp_port1; u_int16_t detected_gnutella_udp_port2; -/* NDPI_PROTOCOL_SOULSEEK */ + /* NDPI_PROTOCOL_SOULSEEK */ u_int16_t soulseek_listen_port; -/* NDPI_PROTOCOL_IRC */ + /* NDPI_PROTOCOL_IRC */ u_int8_t irc_number_of_port; -/* NDPI_PROTOCOL_OSCAR */ + /* NDPI_PROTOCOL_OSCAR */ u_int8_t oscar_ssl_session_id[33]; -/* NDPI_PROTOCOL_UNENCRYPTED_JABBER */ + /* NDPI_PROTOCOL_UNENCRYPTED_JABBER */ u_int8_t jabber_voice_stun_used_ports; -/* NDPI_PROTOCOL_SIP */ -/* NDPI_PROTOCOL_YAHOO */ + /* NDPI_PROTOCOL_SIP */ + /* NDPI_PROTOCOL_YAHOO */ u_int32_t yahoo_video_lan_dir:1; -/* NDPI_PROTOCOL_YAHOO */ + /* NDPI_PROTOCOL_YAHOO */ u_int32_t yahoo_conf_logged_in:1; u_int32_t yahoo_voice_conf_logged_in:1; -/* NDPI_PROTOCOL_RTSP */ + /* NDPI_PROTOCOL_RTSP */ u_int32_t rtsp_ts_set:1; }; /* ************************************************** */ struct ndpi_flow_tcp_struct { -/* NDPI_PROTOCOL_MAIL_SMTP */ + /* NDPI_PROTOCOL_MAIL_SMTP */ u_int16_t smtp_command_bitmask; -/* NDPI_PROTOCOL_MAIL_POP */ + /* NDPI_PROTOCOL_MAIL_POP */ u_int16_t pop_command_bitmask; -/* NDPI_PROTOCOL_QQ */ + /* NDPI_PROTOCOL_QQ */ u_int16_t qq_nxt_len; /* NDPI_PROTOCOL_WHATSAPP */ u_int8_t wa_matched_so_far; -/* NDPI_PROTOCOL_TDS */ + /* NDPI_PROTOCOL_TDS */ u_int8_t tds_login_version; -/* NDPI_PROTOCOL_IRC */ + /* NDPI_PROTOCOL_IRC */ u_int8_t irc_stage; u_int8_t irc_port; -/* NDPI_PROTOCOL_H323 */ + /* NDPI_PROTOCOL_H323 */ u_int8_t h323_valid_packets; -/* NDPI_PROTOCOL_GNUTELLA */ + /* NDPI_PROTOCOL_GNUTELLA */ u_int8_t gnutella_msg_id[3]; -/* NDPI_PROTOCOL_IRC */ + /* NDPI_PROTOCOL_IRC */ u_int32_t irc_3a_counter:3; u_int32_t irc_stage2:5; u_int32_t irc_direction:2; u_int32_t irc_0x1000_full:1; -/* NDPI_PROTOCOL_SOULSEEK */ + /* NDPI_PROTOCOL_SOULSEEK */ u_int32_t soulseek_stage:2; -/* NDPI_PROTOCOL_TDS */ + /* NDPI_PROTOCOL_TDS */ u_int32_t tds_stage:3; -/* NDPI_PROTOCOL_USENET */ + /* NDPI_PROTOCOL_USENET */ u_int32_t usenet_stage:2; -/* NDPI_PROTOCOL_IMESH */ + /* NDPI_PROTOCOL_IMESH */ u_int32_t imesh_stage:4; -/* NDPI_PROTOCOL_HTTP */ + /* NDPI_PROTOCOL_HTTP */ u_int32_t http_setup_dir:2; u_int32_t http_stage:2; u_int32_t http_empty_line_seen:1; u_int32_t http_wait_for_retransmission:1; -/* NDPI_PROTOCOL_GNUTELLA */ + /* NDPI_PROTOCOL_GNUTELLA */ u_int32_t gnutella_stage:2; // 0 - 2 -/* NDPI_CONTENT_MMS */ + /* NDPI_CONTENT_MMS */ u_int32_t mms_stage:2; -/* NDPI_PROTOCOL_YAHOO */ + /* NDPI_PROTOCOL_YAHOO */ u_int32_t yahoo_sip_comm:1; u_int32_t yahoo_http_proxy_stage:2; -/* NDPI_PROTOCOL_MSN */ + /* NDPI_PROTOCOL_MSN */ u_int32_t msn_stage:3; u_int32_t msn_ssl_ft:2; -/* NDPI_PROTOCOL_SSH */ + /* NDPI_PROTOCOL_SSH */ u_int32_t ssh_stage:3; -/* NDPI_PROTOCOL_VNC */ + /* NDPI_PROTOCOL_VNC */ u_int32_t vnc_stage:2; // 0 - 3 -/* NDPI_PROTOCOL_TELNET */ + /* NDPI_PROTOCOL_TELNET */ u_int32_t telnet_stage:2; // 0 - 2 -/* NDPI_PROTOCOL_SSL */ + /* NDPI_PROTOCOL_SSL */ u_int8_t ssl_seen_client_cert:1, ssl_seen_server_cert:1, ssl_stage:2; // 0 - 5 -/* NDPI_PROTOCOL_POSTGRES */ + /* NDPI_PROTOCOL_POSTGRES */ u_int32_t postgres_stage:3; -/* NDPI_PROTOCOL_DIRECT_DOWNLOAD_LINK */ + /* NDPI_PROTOCOL_DIRECT_DOWNLOAD_LINK */ u_int32_t ddlink_server_direction:1; u_int32_t seen_syn:1; u_int32_t seen_syn_ack:1; u_int32_t seen_ack:1; -/* NDPI_PROTOCOL_ICECAST */ + /* NDPI_PROTOCOL_ICECAST */ u_int32_t icecast_stage:1; -/* NDPI_PROTOCOL_DOFUS */ + /* NDPI_PROTOCOL_DOFUS */ u_int32_t dofus_stage:1; -/* NDPI_PROTOCOL_FIESTA */ + /* NDPI_PROTOCOL_FIESTA */ u_int32_t fiesta_stage:2; -/* NDPI_PROTOCOL_WORLDOFWARCRAFT */ + /* NDPI_PROTOCOL_WORLDOFWARCRAFT */ u_int32_t wow_stage:2; -/* NDPI_PROTOCOL_HTTP_APPLICATION_VEOHTV */ + /* NDPI_PROTOCOL_HTTP_APPLICATION_VEOHTV */ u_int32_t veoh_tv_stage:2; -/* NDPI_PROTOCOL_SHOUTCAST */ + /* NDPI_PROTOCOL_SHOUTCAST */ u_int32_t shoutcast_stage:2; -/* NDPI_PROTOCOL_RTP */ + /* NDPI_PROTOCOL_RTP */ u_int32_t rtp_special_packets_seen:1; -/* NDPI_PROTOCOL_MAIL_POP */ + /* NDPI_PROTOCOL_MAIL_POP */ u_int32_t mail_pop_stage:2; -/* NDPI_PROTOCOL_MAIL_IMAP */ + /* NDPI_PROTOCOL_MAIL_IMAP */ u_int32_t mail_imap_stage:3, mail_imap_starttls:2; -/* NDPI_PROTOCOL_SKYPE */ + /* NDPI_PROTOCOL_SKYPE */ u_int8_t skype_packet_id; -/* NDPI_PROTOCOL_CITRIX */ + /* NDPI_PROTOCOL_CITRIX */ u_int8_t citrix_packet_id; -/* NDPI_PROTOCOL_LOTUS_NOTES */ + /* NDPI_PROTOCOL_LOTUS_NOTES */ u_int8_t lotus_notes_packet_id; -/* NDPI_PROTOCOL_TEAMVIEWER */ + /* NDPI_PROTOCOL_TEAMVIEWER */ u_int8_t teamviewer_stage; -/* NDPI_PROTOCOL_ZMQ */ + /* NDPI_PROTOCOL_ZMQ */ u_int8_t prev_zmq_pkt_len; u_char prev_zmq_pkt[10]; -/* NDPI_PROTOCOL_PPSTREAM */ + /* NDPI_PROTOCOL_PPSTREAM */ u_int32_t ppstream_stage:3; -/* NDPI_PROTOCOL_MEMCACHED */ + /* NDPI_PROTOCOL_MEMCACHED */ u_int8_t memcached_matches; -/* NDPI_PROTOCOL_NEST_LOG_SINK */ + /* NDPI_PROTOCOL_NEST_LOG_SINK */ u_int8_t nest_log_sink_matches; } #ifndef WIN32 @@ -608,51 +608,51 @@ struct ndpi_flow_tcp_struct { /* ************************************************** */ struct ndpi_flow_udp_struct { -/* NDPI_PROTOCOL_BATTLEFIELD */ + /* NDPI_PROTOCOL_BATTLEFIELD */ u_int32_t battlefield_msg_id; -/* NDPI_PROTOCOL_SNMP */ + /* NDPI_PROTOCOL_SNMP */ u_int32_t snmp_msg_id; -/* NDPI_PROTOCOL_BATTLEFIELD */ + /* NDPI_PROTOCOL_BATTLEFIELD */ u_int32_t battlefield_stage:3; -/* NDPI_PROTOCOL_SNMP */ + /* NDPI_PROTOCOL_SNMP */ u_int32_t snmp_stage:2; -/* NDPI_PROTOCOL_PPSTREAM */ + /* NDPI_PROTOCOL_PPSTREAM */ u_int32_t ppstream_stage:3; // 0 - 7 -/* NDPI_PROTOCOL_HALFLIFE2 */ + /* NDPI_PROTOCOL_HALFLIFE2 */ u_int32_t halflife2_stage:2; // 0 - 2 -/* NDPI_PROTOCOL_TFTP */ + /* NDPI_PROTOCOL_TFTP */ u_int32_t tftp_stage:1; -/* NDPI_PROTOCOL_AIMINI */ + /* NDPI_PROTOCOL_AIMINI */ u_int32_t aimini_stage:5; -/* NDPI_PROTOCOL_XBOX */ + /* NDPI_PROTOCOL_XBOX */ u_int32_t xbox_stage:1; -/* NDPI_PROTOCOL_WINDOWS_UPDATE */ + /* NDPI_PROTOCOL_WINDOWS_UPDATE */ u_int32_t wsus_stage:1; -/* NDPI_PROTOCOL_SKYPE */ + /* NDPI_PROTOCOL_SKYPE */ u_int8_t skype_packet_id; -/* NDPI_PROTOCOL_TEAMVIEWER */ + /* NDPI_PROTOCOL_TEAMVIEWER */ u_int8_t teamviewer_stage; -/* NDPI_PROTOCOL_EAQ */ + /* NDPI_PROTOCOL_EAQ */ u_int8_t eaq_pkt_id; u_int32_t eaq_sequence; -/* NDPI_PROTOCOL_RX */ + /* NDPI_PROTOCOL_RX */ u_int32_t rx_conn_epoch; u_int32_t rx_conn_id; -/* NDPI_PROTOCOL_MEMCACHED */ + /* NDPI_PROTOCOL_MEMCACHED */ u_int8_t memcached_matches; } #ifndef WIN32 @@ -746,87 +746,87 @@ typedef struct { } ndpi_port_range; typedef enum { - NDPI_PROTOCOL_SAFE = 0, /* Surely doesn't provide risks for the network. (e.g., a news site) */ - NDPI_PROTOCOL_ACCEPTABLE, /* Probably doesn't provide risks, but could be malicious (e.g., Dropbox) */ - NDPI_PROTOCOL_FUN, /* Pure fun protocol, which may be prohibited by the user policy (e.g., Netflix) */ - NDPI_PROTOCOL_UNSAFE, /* Probably provides risks, but could be a normal traffic. Unencrypted protocols with clear pass should be here (e.g., telnet) */ - NDPI_PROTOCOL_POTENTIALLY_DANGEROUS, /* Surely is dangerous (ex. Tor). Be prepared to troubles */ - NDPI_PROTOCOL_TRACKER_ADS, /* Trackers, Advertisements... */ - NDPI_PROTOCOL_UNRATED /* No idea, not implemented or impossible to classify */ + NDPI_PROTOCOL_SAFE = 0, /* Surely doesn't provide risks for the network. (e.g., a news site) */ + NDPI_PROTOCOL_ACCEPTABLE, /* Probably doesn't provide risks, but could be malicious (e.g., Dropbox) */ + NDPI_PROTOCOL_FUN, /* Pure fun protocol, which may be prohibited by the user policy (e.g., Netflix) */ + NDPI_PROTOCOL_UNSAFE, /* Probably provides risks, but could be a normal traffic. Unencrypted protocols with clear pass should be here (e.g., telnet) */ + NDPI_PROTOCOL_POTENTIALLY_DANGEROUS, /* Surely is dangerous (ex. Tor). Be prepared to troubles */ + NDPI_PROTOCOL_TRACKER_ADS, /* Trackers, Advertisements... */ + NDPI_PROTOCOL_UNRATED /* No idea, not implemented or impossible to classify */ } ndpi_protocol_breed_t; #define NUM_BREEDS (NDPI_PROTOCOL_UNRATED+1) /* Abstract categories to group the protocols. */ typedef enum { - NDPI_PROTOCOL_CATEGORY_UNSPECIFIED = 0, /* For general services and unknown protocols */ - NDPI_PROTOCOL_CATEGORY_MEDIA, /* Multimedia and streaming */ - NDPI_PROTOCOL_CATEGORY_VPN, /* Virtual Private Networks */ - NDPI_PROTOCOL_CATEGORY_MAIL, /* Protocols to send/receive/sync emails */ - NDPI_PROTOCOL_CATEGORY_DATA_TRANSFER, /* AFS/NFS and similar protocols */ - NDPI_PROTOCOL_CATEGORY_WEB, /* Web/mobile protocols and services */ - NDPI_PROTOCOL_CATEGORY_SOCIAL_NETWORK, /* Social networks */ - NDPI_PROTOCOL_CATEGORY_DOWNLOAD_FT, /* Download, FTP, file transfer/sharing */ - NDPI_PROTOCOL_CATEGORY_GAME, /* Online games */ - NDPI_PROTOCOL_CATEGORY_CHAT, /* Instant messaging */ - NDPI_PROTOCOL_CATEGORY_VOIP, /* Real-time communications and conferencing */ - NDPI_PROTOCOL_CATEGORY_DATABASE, /* Protocols for database communication */ - NDPI_PROTOCOL_CATEGORY_REMOTE_ACCESS, /* Remote access and control */ - NDPI_PROTOCOL_CATEGORY_CLOUD, /* Online cloud services */ - NDPI_PROTOCOL_CATEGORY_NETWORK, /* Network infrastructure protocols */ - NDPI_PROTOCOL_CATEGORY_COLLABORATIVE, /* Software for collaborative development, including Webmail */ - NDPI_PROTOCOL_CATEGORY_RPC, /* High level network communication protocols */ - NDPI_PROTOCOL_CATEGORY_STREAMING, /* Streaming protocols */ - NDPI_PROTOCOL_CATEGORY_SYSTEM_OS, /* System/Operating System level applications */ - NDPI_PROTOCOL_CATEGORY_SW_UPDATE, /* Software update */ - - /* See #define NUM_CUSTOM_CATEGORIES */ - NDPI_PROTOCOL_CATEGORY_CUSTOM_1, /* User custom category 1 */ - NDPI_PROTOCOL_CATEGORY_CUSTOM_2, /* User custom category 2 */ - NDPI_PROTOCOL_CATEGORY_CUSTOM_3, /* User custom category 3 */ - NDPI_PROTOCOL_CATEGORY_CUSTOM_4, /* User custom category 4 */ - NDPI_PROTOCOL_CATEGORY_CUSTOM_5, /* User custom category 5 */ - - /* Payload Content */ - NDPI_CONTENT_CATEGORY_AVI, - NDPI_CONTENT_CATEGORY_FLASH, - NDPI_CONTENT_CATEGORY_OGG, - NDPI_CONTENT_CATEGORY_MPEG, - NDPI_CONTENT_CATEGORY_QUICKTIME, - NDPI_CONTENT_CATEGORY_REALMEDIA, - NDPI_CONTENT_CATEGORY_WINDOWSMEDIA, - NDPI_CONTENT_CATEGORY_WEBM, - - /* Some custom categories */ - CUSTOM_CATEGORY_MINING = 99, - CUSTOM_CATEGORY_MALWARE = 100, - CUSTOM_CATEGORY_ADVERTISEMENT = 101, - CUSTOM_CATEGORY_BANNED_SITE = 102, - CUSTOM_CATEGORY_SITE_UNAVAILABLE = 103, - - /* - IMPORTANT - - Please keep in sync with - - static const char* categories[] = { ..} - - in ndpi_main.c - */ + NDPI_PROTOCOL_CATEGORY_UNSPECIFIED = 0, /* For general services and unknown protocols */ + NDPI_PROTOCOL_CATEGORY_MEDIA, /* Multimedia and streaming */ + NDPI_PROTOCOL_CATEGORY_VPN, /* Virtual Private Networks */ + NDPI_PROTOCOL_CATEGORY_MAIL, /* Protocols to send/receive/sync emails */ + NDPI_PROTOCOL_CATEGORY_DATA_TRANSFER, /* AFS/NFS and similar protocols */ + NDPI_PROTOCOL_CATEGORY_WEB, /* Web/mobile protocols and services */ + NDPI_PROTOCOL_CATEGORY_SOCIAL_NETWORK, /* Social networks */ + NDPI_PROTOCOL_CATEGORY_DOWNLOAD_FT, /* Download, FTP, file transfer/sharing */ + NDPI_PROTOCOL_CATEGORY_GAME, /* Online games */ + NDPI_PROTOCOL_CATEGORY_CHAT, /* Instant messaging */ + NDPI_PROTOCOL_CATEGORY_VOIP, /* Real-time communications and conferencing */ + NDPI_PROTOCOL_CATEGORY_DATABASE, /* Protocols for database communication */ + NDPI_PROTOCOL_CATEGORY_REMOTE_ACCESS, /* Remote access and control */ + NDPI_PROTOCOL_CATEGORY_CLOUD, /* Online cloud services */ + NDPI_PROTOCOL_CATEGORY_NETWORK, /* Network infrastructure protocols */ + NDPI_PROTOCOL_CATEGORY_COLLABORATIVE, /* Software for collaborative development, including Webmail */ + NDPI_PROTOCOL_CATEGORY_RPC, /* High level network communication protocols */ + NDPI_PROTOCOL_CATEGORY_STREAMING, /* Streaming protocols */ + NDPI_PROTOCOL_CATEGORY_SYSTEM_OS, /* System/Operating System level applications */ + NDPI_PROTOCOL_CATEGORY_SW_UPDATE, /* Software update */ + + /* See #define NUM_CUSTOM_CATEGORIES */ + NDPI_PROTOCOL_CATEGORY_CUSTOM_1, /* User custom category 1 */ + NDPI_PROTOCOL_CATEGORY_CUSTOM_2, /* User custom category 2 */ + NDPI_PROTOCOL_CATEGORY_CUSTOM_3, /* User custom category 3 */ + NDPI_PROTOCOL_CATEGORY_CUSTOM_4, /* User custom category 4 */ + NDPI_PROTOCOL_CATEGORY_CUSTOM_5, /* User custom category 5 */ + + /* Payload Content */ + NDPI_CONTENT_CATEGORY_AVI, + NDPI_CONTENT_CATEGORY_FLASH, + NDPI_CONTENT_CATEGORY_OGG, + NDPI_CONTENT_CATEGORY_MPEG, + NDPI_CONTENT_CATEGORY_QUICKTIME, + NDPI_CONTENT_CATEGORY_REALMEDIA, + NDPI_CONTENT_CATEGORY_WINDOWSMEDIA, + NDPI_CONTENT_CATEGORY_WEBM, + + /* Some custom categories */ + CUSTOM_CATEGORY_MINING = 99, + CUSTOM_CATEGORY_MALWARE = 100, + CUSTOM_CATEGORY_ADVERTISEMENT = 101, + CUSTOM_CATEGORY_BANNED_SITE = 102, + CUSTOM_CATEGORY_SITE_UNAVAILABLE = 103, + + /* + IMPORTANT + + Please keep in sync with + + static const char* categories[] = { ..} + + in ndpi_main.c + */ - NDPI_PROTOCOL_NUM_CATEGORIES /* - NOTE: Keep this as last member - Unused as value but useful to getting the number of elements - in this datastructure - */ + NDPI_PROTOCOL_NUM_CATEGORIES /* + NOTE: Keep this as last member + Unused as value but useful to getting the number of elements + in this datastructure + */ } ndpi_protocol_category_t; typedef enum { - ndpi_pref_http_dont_dissect_response = 0, - ndpi_pref_dns_dissect_response, - ndpi_pref_direction_detect_disable, - ndpi_pref_disable_metadata_export, - ndpi_pref_enable_category_substring_match + ndpi_pref_http_dont_dissect_response = 0, + ndpi_pref_dns_dissect_response, + ndpi_pref_direction_detect_disable, + ndpi_pref_disable_metadata_export, + ndpi_pref_enable_category_substring_match } ndpi_detection_preference; /* ntop extensions */ @@ -875,9 +875,9 @@ typedef struct ndpi_proto { #include struct hs_list { - char *expression; - unsigned int id; - struct hs_list *next; + char *expression; + unsigned int id; + struct hs_list *next; }; struct hs { @@ -986,13 +986,13 @@ struct ndpi_detection_module_struct { char ip_string[NDPI_IP_STRING_SIZE]; #endif u_int8_t ip_version_limit; -/* NDPI_PROTOCOL_BITTORRENT */ + /* NDPI_PROTOCOL_BITTORRENT */ struct hash_ip4p_table *bt_ht; #ifdef NDPI_DETECTION_SUPPORT_IPV6 struct hash_ip4p_table *bt6_ht; #endif -/* BT_ANNOUNCE */ + /* BT_ANNOUNCE */ struct bt_announce *bt_ann; int bt_ann_len; @@ -1122,7 +1122,7 @@ struct ndpi_flow_struct { char fingerprint[48]; char class_ident[48]; } dhcp; - } protos; + } protos; /*** ALL protocol specific 64 bit variables here ***/ @@ -1131,90 +1131,83 @@ struct ndpi_flow_struct { ndpi_protocol_category_t category; -/* NDPI_PROTOCOL_REDIS */ + /* NDPI_PROTOCOL_REDIS */ u_int8_t redis_s2d_first_char, redis_d2s_first_char; u_int16_t packet_counter; // can be 0 - 65000 u_int16_t packet_direction_counter[2]; u_int16_t byte_counter[2]; -/* NDPI_PROTOCOL_BITTORRENT */ + /* NDPI_PROTOCOL_BITTORRENT */ u_int8_t bittorrent_stage; // can be 0 - 255 -/* NDPI_PROTOCOL_DIRECTCONNECT */ - u_int32_t directconnect_stage:2; // 0 - 1 - -/* NDPI_PROTOCOL_YAHOO */ - u_int32_t sip_yahoo_voice:1; + /* NDPI_PROTOCOL_DIRECTCONNECT */ + u_int8_t directconnect_stage:2; // 0 - 1 -/* NDPI_PROTOCOL_HTTP */ - u_int32_t http_detected:1; + /* NDPI_PROTOCOL_YAHOO */ + u_int8_t sip_yahoo_voice:1; -/* NDPI_PROTOCOL_RTSP */ - u_int32_t rtsprdt_stage:2; - u_int32_t rtsp_control_flow:1; + /* NDPI_PROTOCOL_HTTP */ + u_int8_t http_detected:1; + u_int16_t http_upper_protocol, http_lower_protocol; + + /* NDPI_PROTOCOL_RTSP */ + u_int8_t rtsprdt_stage:2, rtsp_control_flow:1; -/* NDPI_PROTOCOL_YAHOO */ - u_int32_t yahoo_detection_finished:2; + /* NDPI_PROTOCOL_YAHOO */ + u_int8_t yahoo_detection_finished:2; -/* NDPI_PROTOCOL_ZATTOO */ - u_int32_t zattoo_stage:3; + /* NDPI_PROTOCOL_ZATTOO */ + u_int8_t zattoo_stage:3; -/* NDPI_PROTOCOL_QQ */ - u_int32_t qq_stage:3; + /* NDPI_PROTOCOL_QQ */ + u_int8_t qq_stage:3; -/* NDPI_PROTOCOL_THUNDER */ - u_int32_t thunder_stage:2; // 0 - 3 + /* NDPI_PROTOCOL_THUNDER */ + u_int8_t thunder_stage:2; // 0 - 3 -/* NDPI_PROTOCOL_OSCAR */ - u_int32_t oscar_ssl_voice_stage:3; - u_int32_t oscar_video_voice:1; + /* NDPI_PROTOCOL_OSCAR */ + u_int8_t oscar_ssl_voice_stage:3, oscar_video_voice:1; -/* NDPI_PROTOCOL_FLORENSIA */ - u_int32_t florensia_stage:1; + /* NDPI_PROTOCOL_FLORENSIA */ + u_int8_t florensia_stage:1; -/* NDPI_PROTOCOL_SOCKS */ - u_int32_t socks5_stage:2; // 0 - 3 - u_int32_t socks4_stage:2; // 0 - 3 + /* NDPI_PROTOCOL_SOCKS */ + u_int8_t socks5_stage:2, socks4_stage:2; // 0 - 3 -/* NDPI_PROTOCOL_EDONKEY */ - u_int32_t edonkey_stage:2; // 0 - 3 + /* NDPI_PROTOCOL_EDONKEY */ + u_int8_t edonkey_stage:2; // 0 - 3 -/* NDPI_PROTOCOL_FTP_CONTROL */ - u_int32_t ftp_control_stage:2; + /* NDPI_PROTOCOL_FTP_CONTROL */ + u_int8_t ftp_control_stage:2; -/* NDPI_PROTOCOL_RTMP */ - u_int32_t rtmp_stage:2; + /* NDPI_PROTOCOL_RTMP */ + u_int8_t rtmp_stage:2; -/* NDPI_PROTOCOL_PANDO */ - u_int32_t pando_stage:3; + /* NDPI_PROTOCOL_PANDO */ + u_int8_t pando_stage:3; -/* NDPI_PROTOCOL_STEAM */ - u_int32_t steam_stage:3; - u_int32_t steam_stage1:3; // 0 - 4 - u_int32_t steam_stage2:2; // 0 - 2 - u_int32_t steam_stage3:2; // 0 - 2 + /* NDPI_PROTOCOL_STEAM */ + u_int16_t steam_stage:3, steam_stage1:3, steam_stage2:2, steam_stage3:2; -/* NDPI_PROTOCOL_PPLIVE */ - u_int32_t pplive_stage1:3; // 0 - 6 - u_int32_t pplive_stage2:2; // 0 - 2 - u_int32_t pplive_stage3:2; // 0 - 2 + /* NDPI_PROTOCOL_PPLIVE */ + u_int8_t pplive_stage1:3, pplive_stage2:2, pplive_stage3:2; -/* NDPI_PROTOCOL_STARCRAFT */ - u_int32_t starcraft_udp_stage : 3; // 0-7 + /* NDPI_PROTOCOL_STARCRAFT */ + u_int8_t starcraft_udp_stage : 3; // 0-7 -/* NDPI_PROTOCOL_OPENVPN */ + /* NDPI_PROTOCOL_OPENVPN */ u_int8_t ovpn_session_id[8]; u_int8_t ovpn_counter; -/* NDPI_PROTOCOL_TINC */ + /* NDPI_PROTOCOL_TINC */ u_int8_t tinc_state; struct tinc_cache_entry tinc_cache_entry; -/* NDPI_PROTOCOL_CSGO */ + /* NDPI_PROTOCOL_CSGO */ u_int8_t csgo_strid[18],csgo_state,csgo_s2; u_int32_t csgo_id2; -/* NDPI_PROTOCOL_1KXUN || NDPI_PROTOCOL_IQIYI */ + /* NDPI_PROTOCOL_1KXUN || NDPI_PROTOCOL_IQIYI */ u_int16_t kxun_counter, iqiyi_counter; /* internal structures to save functions calls */ diff --git a/src/lib/ndpi_main.c b/src/lib/ndpi_main.c index b8d8f358d..863943c4a 100644 --- a/src/lib/ndpi_main.c +++ b/src/lib/ndpi_main.c @@ -3901,8 +3901,7 @@ void check_ndpi_tcp_flow_func(struct ndpi_detection_module_struct *ndpi_struct, if((proto_id != NDPI_PROTOCOL_UNKNOWN) && NDPI_BITMASK_COMPARE(flow->excluded_protocol_bitmask, ndpi_struct->callback_buffer[proto_index].excluded_protocol_bitmask) == 0 - && NDPI_BITMASK_COMPARE(ndpi_struct->callback_buffer[proto_index].detection_bitmask, - detection_bitmask) != 0 + && NDPI_BITMASK_COMPARE(ndpi_struct->callback_buffer[proto_index].detection_bitmask, detection_bitmask) != 0 && (ndpi_struct->callback_buffer[proto_index].ndpi_selection_bitmask & *ndpi_selection_packet) == ndpi_struct->callback_buffer[proto_index].ndpi_selection_bitmask) { if((flow->guessed_protocol_id != NDPI_PROTOCOL_UNKNOWN) && (ndpi_struct->proto_defaults[flow->guessed_protocol_id].func != NULL)) @@ -3920,7 +3919,6 @@ void check_ndpi_tcp_flow_func(struct ndpi_detection_module_struct *ndpi_struct, detection_bitmask) != 0) { ndpi_struct->callback_buffer_tcp_payload[a].func(ndpi_struct, flow); - if(flow->detected_protocol_stack[0] != NDPI_PROTOCOL_UNKNOWN) break; /* Stop after detecting the first protocol */ } @@ -4731,14 +4729,13 @@ void ndpi_parse_packet_line_info(struct ndpi_detection_module_struct *ndpi_struc { u_int32_t a; struct ndpi_packet_struct *packet = &flow->packet; + if(packet->packet_lines_parsed_complete != 0) return; packet->packet_lines_parsed_complete = 1; packet->parsed_lines = 0; - packet->empty_line_position_set = 0; - packet->host_line.ptr = NULL; packet->host_line.len = 0; packet->referer_line.ptr = NULL; @@ -4779,7 +4776,6 @@ void ndpi_parse_packet_line_info(struct ndpi_detection_module_struct *ndpi_struc packet->line[packet->parsed_lines].len = 0; for(a = 0; a < packet->payload_packet_len; a++) { - if((a + 1) == packet->payload_packet_len) return; /* Return if only one byte remains (prevent invalid reads past end-of-buffer) */ @@ -4788,22 +4784,22 @@ void ndpi_parse_packet_line_info(struct ndpi_detection_module_struct *ndpi_struc /* First line of a HTTP response parsing. Expected a "HTTP/1.? ???" */ if(packet->parsed_lines == 0 && packet->line[0].len >= NDPI_STATICSTRING_LEN("HTTP/1.X 200 ") && - strncasecmp((const char *)packet->line[0].ptr, "HTTP/1.", NDPI_STATICSTRING_LEN("HTTP/1.")) == 0 && - packet->line[0].ptr[NDPI_STATICSTRING_LEN("HTTP/1.X ")] > '0' && /* response code between 000 and 699 */ - packet->line[0].ptr[NDPI_STATICSTRING_LEN("HTTP/1.X ")] < '6') { - - packet->http_response.ptr = &packet->line[0].ptr[NDPI_STATICSTRING_LEN("HTTP/1.1 ")]; - packet->http_response.len = packet->line[0].len - NDPI_STATICSTRING_LEN("HTTP/1.1 "); - packet->http_num_headers++; - - /* Set server HTTP response code */ - strncpy((char*)flow->http.response_status_code, (char*)packet->http_response.ptr, 3); - flow->http.response_status_code[4]='\0'; - - NDPI_LOG_DBG2(ndpi_struct, - "ndpi_parse_packet_line_info: HTTP response parsed: \"%.*s\"\n", - packet->http_response.len, packet->http_response.ptr); + strncasecmp((const char *)packet->line[0].ptr, "HTTP/1.", NDPI_STATICSTRING_LEN("HTTP/1.")) == 0 && + packet->line[0].ptr[NDPI_STATICSTRING_LEN("HTTP/1.X ")] > '0' && /* response code between 000 and 699 */ + packet->line[0].ptr[NDPI_STATICSTRING_LEN("HTTP/1.X ")] < '6') { + packet->http_response.ptr = &packet->line[0].ptr[NDPI_STATICSTRING_LEN("HTTP/1.1 ")]; + packet->http_response.len = packet->line[0].len - NDPI_STATICSTRING_LEN("HTTP/1.1 "); + packet->http_num_headers++; + + /* Set server HTTP response code */ + strncpy((char*)flow->http.response_status_code, (char*)packet->http_response.ptr, 3); + flow->http.response_status_code[4] = '\0'; + + NDPI_LOG_DBG2(ndpi_struct, + "ndpi_parse_packet_line_info: HTTP response parsed: \"%.*s\"\n", + packet->http_response.len, packet->http_response.ptr); } + /* "Server:" header line in HTTP response */ if(packet->line[packet->parsed_lines].len > NDPI_STATICSTRING_LEN("Server:") + 1 && strncasecmp((const char *)packet->line[packet->parsed_lines].ptr, "Server:", NDPI_STATICSTRING_LEN("Server:")) == 0) { @@ -5253,9 +5249,8 @@ void ndpi_int_reset_protocol(struct ndpi_flow_struct *flow) { if(flow) { int a; - for(a = 0; a < NDPI_PROTOCOL_SIZE; a++) { - flow->detected_protocol_stack[a] = NDPI_PROTOCOL_UNKNOWN; - } + for(a = 0; a < NDPI_PROTOCOL_SIZE; a++) + flow->detected_protocol_stack[a] = NDPI_PROTOCOL_UNKNOWN; } } diff --git a/src/lib/protocols/apple_push.c b/src/lib/protocols/apple_push.c index 6930dba86..45346e07b 100644 --- a/src/lib/protocols/apple_push.c +++ b/src/lib/protocols/apple_push.c @@ -31,20 +31,22 @@ static void ndpi_check_apple_push(struct ndpi_detection_module_struct *ndpi_stru struct ndpi_flow_struct *flow) { struct ndpi_packet_struct *packet = &flow->packet; - /* https://support.apple.com/en-us/HT203609 */ - if(((ntohl(packet->iph->saddr) & 0xFF000000 /* 255.0.0.0 */) == 0x11000000 /* 17.0.0.0/8 */) - || ((ntohl(packet->iph->daddr) & 0xFF000000 /* 255.0.0.0 */) == 0x11000000 /* 17.0.0.0/8 */)) { - u_int16_t apple_push_port = ntohs(5223); - u_int16_t notification_apn_port = ntohs(2195); - u_int16_t apn_feedback_port = ntohs(2196); - - if(((packet->tcp->source == apple_push_port) || (packet->tcp->dest == apple_push_port)) - || ((packet->tcp->source == notification_apn_port) || (packet->tcp->dest == notification_apn_port)) - || ((packet->tcp->source == apn_feedback_port) || (packet->tcp->dest == apn_feedback_port)) - ) { - NDPI_LOG_INFO(ndpi_struct, "found apple_push\n"); - ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_APPLE_PUSH, NDPI_PROTOCOL_UNKNOWN); - return; + if(packet->iph) { + /* https://support.apple.com/en-us/HT203609 */ + if(((ntohl(packet->iph->saddr) & 0xFF000000 /* 255.0.0.0 */) == 0x11000000 /* 17.0.0.0/8 */) + || ((ntohl(packet->iph->daddr) & 0xFF000000 /* 255.0.0.0 */) == 0x11000000 /* 17.0.0.0/8 */)) { + u_int16_t apple_push_port = ntohs(5223); + u_int16_t notification_apn_port = ntohs(2195); + u_int16_t apn_feedback_port = ntohs(2196); + + if(((packet->tcp->source == apple_push_port) || (packet->tcp->dest == apple_push_port)) + || ((packet->tcp->source == notification_apn_port) || (packet->tcp->dest == notification_apn_port)) + || ((packet->tcp->source == apn_feedback_port) || (packet->tcp->dest == apn_feedback_port)) + ) { + NDPI_LOG_INFO(ndpi_struct, "found apple_push\n"); + ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_APPLE_PUSH, NDPI_PROTOCOL_UNKNOWN); + return; + } } } diff --git a/src/lib/protocols/http.c b/src/lib/protocols/http.c index 09b816129..e92701072 100644 --- a/src/lib/protocols/http.c +++ b/src/lib/protocols/http.c @@ -28,27 +28,41 @@ #include "ndpi_api.h" #include "lruc.h" -/* global variables used for 1kxun protocol and iqiyi service */ - static void ndpi_int_http_add_connection(struct ndpi_detection_module_struct *ndpi_struct, struct ndpi_flow_struct *flow, u_int16_t category) { - +#ifdef DEBUG + printf("[%s] [http_dont_dissect_response: %u]->> %s\n", __FUNCTION__, + ndpi_struct->http_dont_dissect_response, flow->http.response_status_code); +#endif + if(flow->detected_protocol_stack[0] == NDPI_PROTOCOL_UNKNOWN) { /* This is HTTP and it is not a sub protocol (e.g. skype or dropbox) */ ndpi_search_tcp_or_udp(ndpi_struct, flow); /* If no custom protocol has been detected */ - if(flow->guessed_host_protocol_id != NDPI_PROTOCOL_UNKNOWN) { ndpi_int_reset_protocol(flow); - ndpi_set_detected_protocol(ndpi_struct, flow, flow->guessed_host_protocol_id, NDPI_PROTOCOL_HTTP); + flow->http_upper_protocol = flow->guessed_host_protocol_id, flow->http_lower_protocol = NDPI_PROTOCOL_HTTP; } else - ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_HTTP, NDPI_PROTOCOL_UNKNOWN); - - flow->http_detected = 1, flow->guessed_category = category; + flow->http_upper_protocol = NDPI_PROTOCOL_HTTP, flow->http_lower_protocol = NDPI_PROTOCOL_UNKNOWN; + + if(ndpi_struct->http_dont_dissect_response) + ndpi_set_detected_protocol(ndpi_struct, flow, flow->http_upper_protocol, flow->http_lower_protocol); + else { + flow->detected_protocol_stack[0] = NDPI_PROTOCOL_UNKNOWN, flow->detected_protocol_stack[1] = NDPI_PROTOCOL_UNKNOWN; + flow->packet.detected_protocol_stack[0] = NDPI_PROTOCOL_UNKNOWN, flow->packet.detected_protocol_stack[1] = NDPI_PROTOCOL_UNKNOWN; + } + } else { + if((!ndpi_struct->http_dont_dissect_response) && (flow->http.response_status_code[0] == '\0')) { + flow->http_upper_protocol = flow->detected_protocol_stack[0], flow->http_lower_protocol = flow->detected_protocol_stack[1]; + flow->detected_protocol_stack[0] = NDPI_PROTOCOL_UNKNOWN, flow->detected_protocol_stack[1] = NDPI_PROTOCOL_UNKNOWN; + flow->packet.detected_protocol_stack[0] = NDPI_PROTOCOL_UNKNOWN, flow->packet.detected_protocol_stack[1] = NDPI_PROTOCOL_UNKNOWN; + } } + + flow->http_detected = 1, flow->guessed_category = category; } #ifdef NDPI_CONTENT_CATEGORY_FLASH @@ -79,7 +93,7 @@ static void avi_check_http_payload(struct ndpi_detection_module_struct *ndpi_str NDPI_LOG_DBG2(ndpi_struct, "called avi_check_http_payload: %u %u %u\n", - packet->empty_line_position_set, flow->l4.tcp.http_empty_line_seen, packet->empty_line_position); + packet->empty_line_position_set, flow->l4.tcp.http_empty_line_seen, packet->empty_line_position); if(packet->empty_line_position_set == 0 && flow->l4.tcp.http_empty_line_seen == 0) return; @@ -125,7 +139,7 @@ static void teamviewer_check_http_payload(struct ndpi_detection_module_struct *n const u_int8_t *pos; NDPI_LOG_DBG2(ndpi_struct, "called teamviewer_check_http_payload: %u %u %u\n", - packet->empty_line_position_set, flow->l4.tcp.http_empty_line_seen, packet->empty_line_position); + packet->empty_line_position_set, flow->l4.tcp.http_empty_line_seen, packet->empty_line_position); if(packet->empty_line_position_set == 0 || (packet->empty_line_position + 5) > (packet->payload_packet_len)) return; @@ -171,10 +185,10 @@ static void setHttpUserAgent(struct ndpi_detection_module_struct *ndpi_struct, static void parseHttpSubprotocol(struct ndpi_detection_module_struct *ndpi_struct, struct ndpi_flow_struct *flow) { if((flow->l4.tcp.http_stage == 0) || (flow->http.url && flow->http_detected)) { - char *double_col = strchr((char*)flow->host_server_name, ':'); - ndpi_protocol_match_result ret_match; - - if(double_col) double_col[0] = '\0'; + char *double_col = strchr((char*)flow->host_server_name, ':'); + ndpi_protocol_match_result ret_match; + + if(double_col) double_col[0] = '\0'; /** NOTE @@ -198,6 +212,13 @@ static void check_content_type_and_change_protocol(struct ndpi_detection_module_ struct ndpi_packet_struct *packet = &flow->packet; u_int8_t a; + if((!ndpi_struct->http_dont_dissect_response) && flow->http_detected && (flow->http.response_status_code[0] != 0)) { + ndpi_set_detected_protocol(ndpi_struct, flow, flow->http_upper_protocol, flow->http_lower_protocol); +#ifdef DEBUG + printf("[%s] [http_dont_dissect_response: %u]->> %s\n", __FUNCTION__, ndpi_struct->http_dont_dissect_response, flow->http.response_status_code); +#endif + return; + } #if defined(NDPI_PROTOCOL_1KXUN) || defined(NDPI_PROTOCOL_IQIYI) /* PPStream */ @@ -277,8 +298,8 @@ static void check_content_type_and_change_protocol(struct ndpi_detection_module_ if(packet->user_agent_line.ptr != NULL && packet->user_agent_line.len != 0) { /** Format examples: - Mozilla/5.0 (iPad; U; CPU OS 3_2 like Mac OS X; en-us) AppleWebKit/531.21.10 (KHTML, like Gecko) .... - Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:54.0) Gecko/20100101 Firefox/54.0 + Mozilla/5.0 (iPad; U; CPU OS 3_2 like Mac OS X; en-us) AppleWebKit/531.21.10 (KHTML, like Gecko) .... + Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:54.0) Gecko/20100101 Firefox/54.0 */ if(packet->user_agent_line.len > 7) { char ua[256]; @@ -308,22 +329,22 @@ static void check_content_type_and_change_protocol(struct ndpi_detection_module_ if(token && (token[0] == ' ')) token++; /* Skip space */ if(token - && ((strcmp(token, "U") == 0) - || (strncmp(token, "MSIE", 4) == 0))) { - token = strsep(&parent, ";"); - if(token && (token[0] == ' ')) token++; /* Skip space */ + && ((strcmp(token, "U") == 0) + || (strncmp(token, "MSIE", 4) == 0))) { + token = strsep(&parent, ";"); + if(token && (token[0] == ' ')) token++; /* Skip space */ - if(token && (strncmp(token, "Update", 6) == 0)) { - token = strsep(&parent, ";"); + if(token && (strncmp(token, "Update", 6) == 0)) { + token = strsep(&parent, ";"); - if(token && (token[0] == ' ')) token++; /* Skip space */ + if(token && (token[0] == ' ')) token++; /* Skip space */ - if(token && (strncmp(token, "AOL", 3) == 0)) { + if(token && (strncmp(token, "AOL", 3) == 0)) { - token = strsep(&parent, ";"); - if(token && (token[0] == ' ')) token++; /* Skip space */ - } - } + token = strsep(&parent, ";"); + if(token && (token[0] == ' ')) token++; /* Skip space */ + } + } } } @@ -340,7 +361,7 @@ static void check_content_type_and_change_protocol(struct ndpi_detection_module_ } NDPI_LOG_DBG2(ndpi_struct, "User Agent Type line found %.*s\n", - packet->user_agent_line.len, packet->user_agent_line.ptr); + packet->user_agent_line.len, packet->user_agent_line.ptr); } /* check for host line */ @@ -348,37 +369,37 @@ static void check_content_type_and_change_protocol(struct ndpi_detection_module_ u_int len; NDPI_LOG_DBG2(ndpi_struct, "HOST line found %.*s\n", - packet->host_line.len, packet->host_line.ptr); + packet->host_line.len, packet->host_line.ptr); /* call ndpi_match_host_subprotocol to see if there is a match with known-host HTTP subprotocol */ if((ndpi_struct->http_dont_dissect_response) || flow->http_detected) { ndpi_protocol_match_result ret_match; - + ndpi_match_host_subprotocol(ndpi_struct, flow, (char*)packet->host_line.ptr, packet->host_line.len, &ret_match, NDPI_PROTOCOL_HTTP); } - + /* Copy result for nDPI apps */ if(!ndpi_struct->disable_metadata_export) { len = ndpi_min(packet->host_line.len, sizeof(flow->host_server_name)-1); strncpy((char*)flow->host_server_name, (char*)packet->host_line.ptr, len); flow->host_server_name[len] = '\0'; } - + flow->server_id = flow->dst; if(packet->forwarded_line.ptr) { - len = ndpi_min(packet->forwarded_line.len, sizeof(flow->protos.http.nat_ip)-1); - if(!ndpi_struct->disable_metadata_export) { - strncpy((char*)flow->protos.http.nat_ip, (char*)packet->forwarded_line.ptr, len); - flow->protos.http.nat_ip[len] = '\0'; - } + len = ndpi_min(packet->forwarded_line.len, sizeof(flow->protos.http.nat_ip)-1); + if(!ndpi_struct->disable_metadata_export) { + strncpy((char*)flow->protos.http.nat_ip, (char*)packet->forwarded_line.ptr, len); + flow->protos.http.nat_ip[len] = '\0'; + } } - if(ndpi_struct->http_dont_dissect_response) + if(!ndpi_struct->http_dont_dissect_response) parseHttpSubprotocol(ndpi_struct, flow); /** @@ -403,26 +424,28 @@ static void check_content_type_and_change_protocol(struct ndpi_detection_module_ && ((ndpi_struct->http_dont_dissect_response) || flow->http_detected) && (packet->http_origin.len > 0)) { ndpi_protocol_match_result ret_match; - + ndpi_match_host_subprotocol(ndpi_struct, flow, (char *)packet->http_origin.ptr, packet->http_origin.len, &ret_match, NDPI_PROTOCOL_HTTP); } - + if(flow->detected_protocol_stack[0] != NDPI_PROTOCOL_UNKNOWN) { if(packet->detected_protocol_stack[0] != NDPI_PROTOCOL_HTTP) { - NDPI_LOG_INFO(ndpi_struct, "found HTTP/%s\n", - ndpi_get_proto_name(ndpi_struct, packet->detected_protocol_stack[0])); + NDPI_LOG_INFO(ndpi_struct, "found HTTP/%s\n", + ndpi_get_proto_name(ndpi_struct, packet->detected_protocol_stack[0])); ndpi_int_http_add_connection(ndpi_struct, flow, packet->detected_protocol_stack[0]); return; /* We have identified a sub-protocol so we're done */ } } } +#if 0 if(!ndpi_struct->http_dont_dissect_response && flow->http_detected) parseHttpSubprotocol(ndpi_struct, flow); +#endif if(flow->guessed_protocol_id == NDPI_PROTOCOL_UNKNOWN) flow->guessed_protocol_id = NDPI_PROTOCOL_HTTP; @@ -430,7 +453,7 @@ static void check_content_type_and_change_protocol(struct ndpi_detection_module_ /* check for accept line */ if(packet->accept_line.ptr != NULL) { NDPI_LOG_DBG2(ndpi_struct, "Accept line found %.*s\n", - packet->accept_line.len, packet->accept_line.ptr); + packet->accept_line.len, packet->accept_line.ptr); if(NDPI_COMPARE_PROTOCOL_TO_BITMASK(ndpi_struct->detection_bitmask, NDPI_PROTOCOL_RTSP) != 0) { rtsp_parse_packet_acceptline(ndpi_struct, flow); @@ -438,26 +461,28 @@ static void check_content_type_and_change_protocol(struct ndpi_detection_module_ } /* search for line startin with "Icy-MetaData" */ - for (a = 0; a < packet->parsed_lines; a++) { + for (a = 0; a < packet->parsed_lines; a++) { if(packet->line[a].len > 11 && memcmp(packet->line[a].ptr, "Icy-MetaData", 12) == 0) { - NDPI_LOG_INFO(ndpi_struct, "found MPEG: Icy-MetaData\n"); - ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_CONTENT_CATEGORY_MPEG); - return; - } + NDPI_LOG_INFO(ndpi_struct, "found MPEG: Icy-MetaData\n"); + ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_CONTENT_CATEGORY_MPEG); + return; + } } if(packet->content_line.ptr != NULL && packet->content_line.len != 0) { NDPI_LOG_DBG2(ndpi_struct, "Content Type line found %.*s\n", - packet->content_line.len, packet->content_line.ptr); + packet->content_line.len, packet->content_line.ptr); if((ndpi_struct->http_dont_dissect_response) || flow->http_detected) { ndpi_protocol_match_result ret_match; - + ndpi_match_content_subprotocol(ndpi_struct, flow, (char*)packet->content_line.ptr, packet->content_line.len, &ret_match, NDPI_PROTOCOL_HTTP); } } + + ndpi_int_http_add_connection(ndpi_struct, flow, packet->detected_protocol_stack[0]); } static void check_http_payload(struct ndpi_detection_module_struct *ndpi_struct, struct ndpi_flow_struct *flow) @@ -486,22 +511,22 @@ static void check_http_payload(struct ndpi_detection_module_struct *ndpi_struct, #define STATIC_STRING_L(a) {.str=a, .len=sizeof(a)-1 } static struct l_string { - const char *str; - size_t len; + const char *str; + size_t len; } http_methods[] = { - STATIC_STRING_L("GET "), - STATIC_STRING_L("POST "), - STATIC_STRING_L("OPTIONS "), - STATIC_STRING_L("HEAD "), - STATIC_STRING_L("PUT "), - STATIC_STRING_L("DELETE "), - STATIC_STRING_L("CONNECT "), - STATIC_STRING_L("PROPFIND "), - STATIC_STRING_L("REPORT ") }; + STATIC_STRING_L("GET "), + STATIC_STRING_L("POST "), + STATIC_STRING_L("OPTIONS "), + STATIC_STRING_L("HEAD "), + STATIC_STRING_L("PUT "), + STATIC_STRING_L("DELETE "), + STATIC_STRING_L("CONNECT "), + STATIC_STRING_L("PROPFIND "), + STATIC_STRING_L("REPORT ") }; static const char *http_fs = "CDGHOPR"; static uint8_t non_ctrl(uint8_t c) { - return c < 32 ? '.':c; + return c < 32 ? '.':c; } static u_int16_t http_request_url_offset(struct ndpi_detection_module_struct *ndpi_struct, struct ndpi_flow_struct *flow) @@ -510,9 +535,9 @@ static u_int16_t http_request_url_offset(struct ndpi_detection_module_struct *nd int i; NDPI_LOG_DBG2(ndpi_struct, "====>>>> HTTP: %c%c%c%c [len: %u]\n", - non_ctrl(packet->payload[0]), non_ctrl(packet->payload[1]), - non_ctrl(packet->payload[2]), non_ctrl(packet->payload[3]), - packet->payload_packet_len); + non_ctrl(packet->payload[0]), non_ctrl(packet->payload[1]), + non_ctrl(packet->payload[2]), non_ctrl(packet->payload[3]), + packet->payload_packet_len); /* Check first char */ if(!strchr(http_fs,packet->payload[0])) return 0; @@ -520,11 +545,11 @@ static u_int16_t http_request_url_offset(struct ndpi_detection_module_struct *nd FIRST PAYLOAD PACKET FROM CLIENT **/ for(i=0; i < sizeof(http_methods)/sizeof(http_methods[0]); i++) { - if(packet->payload_packet_len >= http_methods[i].len && - memcmp(packet->payload,http_methods[i].str,http_methods[i].len) == 0) { - NDPI_LOG_DBG2(ndpi_struct, "HTTP: %sFOUND\n",http_methods[i].str); - return http_methods[i].len; - } + if(packet->payload_packet_len >= http_methods[i].len && + memcmp(packet->payload,http_methods[i].str,http_methods[i].len) == 0) { + NDPI_LOG_DBG2(ndpi_struct, "HTTP: %sFOUND\n",http_methods[i].str); + return http_methods[i].len; + } } return 0; } @@ -556,7 +581,7 @@ static void http_bitmask_exclude_other(struct ndpi_flow_struct *flow) /*************************************************************************************************/ static void ndpi_check_http_tcp(struct ndpi_detection_module_struct *ndpi_struct, - struct ndpi_flow_struct *flow) { + struct ndpi_flow_struct *flow) { struct ndpi_packet_struct *packet = &flow->packet; u_int16_t filename_start; /* the filename in the request method line, e.g., "GET filename_start..."*/ @@ -586,7 +611,7 @@ static void ndpi_check_http_tcp(struct ndpi_detection_module_struct *ndpi_struct flow->l4.tcp.http_stage = 1; return; } - + if((packet->payload_packet_len == 40) && (flow->l4.tcp.http_stage == 0)) { /* -> QR O06L0072-6L91-4O43-857J-K8OO172L6L51 @@ -600,11 +625,11 @@ static void ndpi_check_http_tcp(struct ndpi_detection_module_struct *ndpi_struct && (packet->payload[21] == '-') && (packet->payload[26] == '-') && (packet->payload[39] == 0x0A) - ) - flow->l4.tcp.http_stage = 1; - return; + ) + flow->l4.tcp.http_stage = 1; + return; } - + if((packet->payload_packet_len == 23) && (memcmp(packet->payload, "", 23) == 0)) { /* @@ -617,8 +642,8 @@ static void ndpi_check_http_tcp(struct ndpi_detection_module_struct *ndpi_struct ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_OOKLA, NDPI_PROTOCOL_UNKNOWN); if(ndpi_struct->ookla_cache == NULL) - ndpi_struct->ookla_cache = lruc_new(4*1024, 1024); - + ndpi_struct->ookla_cache = lruc_new(4*1024, 1024); + if(ndpi_struct->ookla_cache != NULL) { u_int8_t *dummy = (u_int8_t*)ndpi_malloc(sizeof(u_int8_t)); @@ -629,23 +654,23 @@ static void ndpi_check_http_tcp(struct ndpi_detection_module_struct *ndpi_struct lruc_set((lruc*)ndpi_struct->ookla_cache, (void*)&packet->iph->daddr, 4, dummy, 1); } } - + return; } - + NDPI_EXCLUDE_PROTO(ndpi_struct, flow); http_bitmask_exclude_other(flow); return; } NDPI_LOG_DBG2(ndpi_struct, - "Filename HTTP found: %d, we look for line info..\n", filename_start); + "Filename HTTP found: %d, we look for line info..\n", filename_start); ndpi_parse_packet_line_info(ndpi_struct, flow); if(packet->parsed_lines <= 1) { NDPI_LOG_DBG2(ndpi_struct, - "Found just one line, we will look further for the next packet...\n"); + "Found just one line, we will look further for the next packet...\n"); packet->http_method.ptr = packet->line[0].ptr; packet->http_method.len = filename_start - 1; @@ -656,10 +681,10 @@ static void ndpi_check_http_tcp(struct ndpi_detection_module_struct *ndpi_struct } NDPI_LOG_DBG2(ndpi_struct, - "Found more than one line, we look further for the next packet...\n"); + "Found more than one line, we look further for the next packet...\n"); if(packet->line[0].len >= (9 + filename_start) - && memcmp(&packet->line[0].ptr[packet->line[0].len - 9], " HTTP/1.", 8) == 0) { /* Request line complete. Ex. "GET / HTTP/1.1" */ + && memcmp(&packet->line[0].ptr[packet->line[0].len - 9], " HTTP/1.", 8) == 0) { /* Request line complete. Ex. "GET / HTTP/1.1" */ packet->http_url_name.ptr = &packet->payload[filename_start]; packet->http_url_name.len = packet->line[0].len - (filename_start + 9); @@ -669,14 +694,13 @@ static void ndpi_check_http_tcp(struct ndpi_detection_module_struct *ndpi_struct // Set the HTTP requested version: 0=HTTP/1.0 and 1=HTTP/1.1 if(memcmp(&packet->line[0].ptr[packet->line[0].len - 1], "1", 1) == 0) - flow->http.request_version = 1; + flow->http.request_version = 1; else - flow->http.request_version = 0; + flow->http.request_version = 0; /* Set the first found headers in request */ flow->http.num_request_headers = packet->http_num_headers; - /* Check for Ookla */ if((packet->referer_line.len > 0) && ndpi_strnstr((const char *)packet->referer_line.ptr, "www.speedtest.net", packet->referer_line.len)) { @@ -686,22 +710,22 @@ static void ndpi_check_http_tcp(struct ndpi_detection_module_struct *ndpi_struct /* Check for additional field introduced by Steam */ int x = 1; if(packet->line[x].len >= 11 && (memcmp(packet->line[x].ptr, "x-steam-sid", 11)) == 0) { - NDPI_LOG_INFO(ndpi_struct, "found STEAM\n"); - ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_STEAM); - check_content_type_and_change_protocol(ndpi_struct, flow); - return; + NDPI_LOG_INFO(ndpi_struct, "found STEAM\n"); + ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_STEAM); + check_content_type_and_change_protocol(ndpi_struct, flow); + return; } /* Check for additional field introduced by Facebook */ x = 1; while(packet->line[x].len != 0) { - if(packet->line[x].len >= 12 && (memcmp(packet->line[x].ptr, "X-FB-SIM-HNI", 12)) == 0) { - NDPI_LOG_INFO(ndpi_struct, "found FACEBOOK\n"); - ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_FACEBOOK); - check_content_type_and_change_protocol(ndpi_struct, flow); - return; - } - x++; + if(packet->line[x].len >= 12 && (memcmp(packet->line[x].ptr, "X-FB-SIM-HNI", 12)) == 0) { + NDPI_LOG_INFO(ndpi_struct, "found FACEBOOK\n"); + ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_FACEBOOK); + check_content_type_and_change_protocol(ndpi_struct, flow); + return; + } + x++; } #if defined(NDPI_PROTOCOL_1KXUN) || defined(NDPI_PROTOCOL_IQIYI) @@ -746,23 +770,23 @@ static void ndpi_check_http_tcp(struct ndpi_detection_module_struct *ndpi_struct } } #endif - + if((packet->http_url_name.len > 7) - && (!strncmp((const char*) packet->http_url_name.ptr, "http://", 7))) { + && (!strncmp((const char*) packet->http_url_name.ptr, "http://", 7))) { NDPI_LOG_INFO(ndpi_struct, "found HTTP_PROXY\n"); ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_HTTP_PROXY); check_content_type_and_change_protocol(ndpi_struct, flow); } if(filename_start == 8 && (memcmp(packet->payload, "CONNECT ", 8) == 0)) { - /* nathan@getoffmalawn.com */ + /* nathan@getoffmalawn.com */ NDPI_LOG_INFO(ndpi_struct, "found HTTP_CONNECT\n"); ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_HTTP_CONNECT); check_content_type_and_change_protocol(ndpi_struct, flow); } NDPI_LOG_DBG2(ndpi_struct, - "HTTP START Found, we will look for sub-protocols (content and host)...\n"); + "HTTP START Found, we will look for sub-protocols (content and host)...\n"); if(packet->host_line.ptr != NULL) { /** @@ -775,11 +799,11 @@ static void ndpi_check_http_tcp(struct ndpi_detection_module_struct *ndpi_struct if(ndpi_struct->http_dont_dissect_response) { if(flow->detected_protocol_stack[0] == NDPI_PROTOCOL_UNKNOWN) /* No subprotocol found */ NDPI_LOG_INFO(ndpi_struct, "found HTTP\n"); - ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_HTTP); + ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_HTTP); } else { flow->http_detected = 1; NDPI_LOG_DBG2(ndpi_struct, - "HTTP START Found, we will look further for the response...\n"); + "HTTP START Found, we will look further for the response...\n"); flow->l4.tcp.http_stage = packet->packet_direction + 1; // packet_direction 0: stage 1, packet_direction 1: stage 2 } @@ -790,23 +814,21 @@ static void ndpi_check_http_tcp(struct ndpi_detection_module_struct *ndpi_struct NDPI_EXCLUDE_PROTO(ndpi_struct, flow); http_bitmask_exclude_other(flow); - } else if((flow->l4.tcp.http_stage == 1) || (flow->l4.tcp.http_stage == 2)) { - NDPI_LOG_DBG2(ndpi_struct, "HTTP stage %u: \n", flow->l4.tcp.http_stage); - + if((packet->payload_packet_len == 34) && (flow->l4.tcp.http_stage == 1)) { if((packet->payload[5] == ' ') && (packet->payload[9] == ' ')) { goto ookla_found; } } - + if((packet->payload_packet_len > 6) && memcmp(packet->payload, "HELLO ", 6) == 0) { - /* This looks like Ookla */ + /* This looks like Ookla */ goto ookla_found; } else - NDPI_ADD_PROTOCOL_TO_BITMASK(flow->excluded_protocol_bitmask, NDPI_PROTOCOL_OOKLA); - + NDPI_ADD_PROTOCOL_TO_BITMASK(flow->excluded_protocol_bitmask, NDPI_PROTOCOL_OOKLA); + /** At first check, if this is for sure a response packet (in another direction. If not, if HTTP is detected do nothing now and return, otherwise check the second packet for the HTTP request @@ -817,7 +839,7 @@ static void ndpi_check_http_tcp(struct ndpi_detection_module_struct *ndpi_struct return; NDPI_LOG_DBG2(ndpi_struct, - " SECOND PAYLOAD TRAFFIC FROM CLIENT, FIRST PACKET MIGHT HAVE BEEN HTTP...UNKNOWN TRAFFIC, HERE FOR HTTP again.. \n"); + " SECOND PAYLOAD TRAFFIC FROM CLIENT, FIRST PACKET MIGHT HAVE BEEN HTTP...UNKNOWN TRAFFIC, HERE FOR HTTP again.. \n"); ndpi_parse_packet_line_info(ndpi_struct, flow); @@ -839,14 +861,14 @@ static void ndpi_check_http_tcp(struct ndpi_detection_module_struct *ndpi_struct } // http://www.slideshare.net/DSPIP/rtsp-analysis-wireshark if(packet->line[0].len >= 9 - && memcmp(&packet->line[0].ptr[packet->line[0].len - 9], " HTTP/1.", 8) == 0) { + && memcmp(&packet->line[0].ptr[packet->line[0].len - 9], " HTTP/1.", 8) == 0) { NDPI_LOG_INFO(ndpi_struct, "found HTTP\n"); ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_HTTP); check_content_type_and_change_protocol(ndpi_struct, flow); NDPI_LOG_DBG2(ndpi_struct, - "HTTP START Found in 2. packet, we will look further for the response....\n"); + "HTTP START Found in 2. packet, we will look further for the response....\n"); flow->http_detected = 1; } @@ -879,9 +901,8 @@ static void ndpi_check_http_tcp(struct ndpi_detection_module_struct *ndpi_struct ndpi_parse_packet_line_info(ndpi_struct, flow); check_content_type_and_change_protocol(ndpi_struct, flow); - if(packet->packet_direction == 1 /* server -> client */){ - flow->http.num_response_headers += packet->http_num_headers; /* flow structs are initialized with zeros */ - } + if(packet->packet_direction == 1 /* server -> client */) + flow->http.num_response_headers += packet->http_num_headers; /* flow structs are initialized with zeros */ if(packet->empty_line_position_set != 0 || flow->l4.tcp.http_empty_line_seen == 1) { NDPI_LOG_DBG2(ndpi_struct, "empty line. check_http_payload\n"); @@ -905,8 +926,8 @@ void ndpi_search_http_tcp(struct ndpi_detection_module_struct *ndpi_struct, } if(packet->detected_protocol_stack[0] != NDPI_PROTOCOL_UNKNOWN) { - return; - } + return; + } NDPI_LOG_DBG(ndpi_struct, "search HTTP\n"); ndpi_check_http_tcp(ndpi_struct, flow); @@ -935,7 +956,7 @@ char* ndpi_get_http_url(struct ndpi_detection_module_struct *ndpi_mod, /* ********************************* */ char* ndpi_get_http_content_type(struct ndpi_detection_module_struct *ndpi_mod, - struct ndpi_flow_struct *flow) { + struct ndpi_flow_struct *flow) { if((!flow) || (!flow->http.content_type)) return(""); else -- cgit v1.2.3 From 396f794bf47178f0afc983eee3cd4a7f3d1fe686 Mon Sep 17 00:00:00 2001 From: u-devel <36368802+u-devel@users.noreply.github.com> Date: Sat, 12 Jan 2019 23:49:24 +0600 Subject: Update radius.c to RFC2865 According to RFC2865 code can have value up to 13, also payload length ranges should be applied. --- src/lib/protocols/radius.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'src/lib') diff --git a/src/lib/protocols/radius.c b/src/lib/protocols/radius.c index 1c85f48d7..209e71177 100644 --- a/src/lib/protocols/radius.c +++ b/src/lib/protocols/radius.c @@ -39,10 +39,13 @@ static void ndpi_check_radius(struct ndpi_detection_module_struct *ndpi_struct, if(packet->udp != NULL) { struct radius_header *h = (struct radius_header*)packet->payload; + /* RFC2865: The minimum length is 20 and maximum length is 4096. */ + if((payload_len < 20) || (payload_len > 4096)) + return; if((payload_len > sizeof(struct radius_header)) && (h->code > 0) - && (h->code <= 5) + && (h->code <= 13) && (ntohs(h->len) == payload_len)) { NDPI_LOG_INFO(ndpi_struct, "Found radius\n"); ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_RADIUS, NDPI_PROTOCOL_UNKNOWN); -- cgit v1.2.3 From 1d155ab2ebb29f30e830713c1b8eefecd03a16d1 Mon Sep 17 00:00:00 2001 From: u-devel <36368802+u-devel@users.noreply.github.com> Date: Sun, 13 Jan 2019 00:07:45 +0600 Subject: FIX H.323 broken detection TPKT header length field can have value more then 255, and in fact in all of the cases I've met it is more then 255. Thus checking real H.323 packet like this: (packet->payload[2] == 0x00) stop detecting H.323 conversation at all. --- src/lib/protocols/h323.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'src/lib') diff --git a/src/lib/protocols/h323.c b/src/lib/protocols/h323.c index ddbcdadf3..d407c981b 100644 --- a/src/lib/protocols/h323.c +++ b/src/lib/protocols/h323.c @@ -31,8 +31,7 @@ void ndpi_search_h323(struct ndpi_detection_module_struct *ndpi_struct, struct n /* H323 */ if(packet->payload_packet_len >= 3 && (packet->payload[0] == 0x03) - && (packet->payload[1] == 0x00) - && (packet->payload[2] == 0x00)) { + && (packet->payload[1] == 0x00)) { struct tpkt *t = (struct tpkt*)packet->payload; u_int16_t len = ntohs(t->len); -- cgit v1.2.3 From f2f2dbe381c0f92f0cc859835238723fea77c767 Mon Sep 17 00:00:00 2001 From: Luca Deri Date: Mon, 14 Jan 2019 20:03:06 +0100 Subject: Fix for IPv6 HTTP traffic --- src/lib/protocols/http.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/lib') diff --git a/src/lib/protocols/http.c b/src/lib/protocols/http.c index e92701072..b1917281c 100644 --- a/src/lib/protocols/http.c +++ b/src/lib/protocols/http.c @@ -630,7 +630,7 @@ static void ndpi_check_http_tcp(struct ndpi_detection_module_struct *ndpi_struct return; } - if((packet->payload_packet_len == 23) && (memcmp(packet->payload, "", 23) == 0)) { + if((packet->iph != NULL) && (packet->payload_packet_len == 23) && (memcmp(packet->payload, "", 23) == 0)) { /* -- cgit v1.2.3 From 2ade0be365429a404d63f8ee20f1b59650cde710 Mon Sep 17 00:00:00 2001 From: Simone Mainardi Date: Tue, 15 Jan 2019 09:56:33 +0100 Subject: Fixes ookla lru cache use with IPv6 --- src/lib/protocols/http.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src/lib') diff --git a/src/lib/protocols/http.c b/src/lib/protocols/http.c index b1917281c..661e55732 100644 --- a/src/lib/protocols/http.c +++ b/src/lib/protocols/http.c @@ -630,7 +630,7 @@ static void ndpi_check_http_tcp(struct ndpi_detection_module_struct *ndpi_struct return; } - if((packet->iph != NULL) && (packet->payload_packet_len == 23) && (memcmp(packet->payload, "", 23) == 0)) { + if((packet->payload_packet_len == 23) && (memcmp(packet->payload, "", 23) == 0)) { /* @@ -644,7 +644,7 @@ static void ndpi_check_http_tcp(struct ndpi_detection_module_struct *ndpi_struct if(ndpi_struct->ookla_cache == NULL) ndpi_struct->ookla_cache = lruc_new(4*1024, 1024); - if(ndpi_struct->ookla_cache != NULL) { + if(packet->iph != NULL && ndpi_struct->ookla_cache != NULL) { u_int8_t *dummy = (u_int8_t*)ndpi_malloc(sizeof(u_int8_t)); if(dummy) { -- cgit v1.2.3 From 0cb4bae27503d2fa6e9e2efd1944049338d7c3c4 Mon Sep 17 00:00:00 2001 From: Ludovico Cavedon Date: Tue, 15 Jan 2019 23:10:48 -0800 Subject: QUIC: convert little endian offsets to host endianness Offset in the QUIC protocol are little endian. Reading them as uint32_t works on little endian architectures, but breaks on big endian ones. This change applies the proper conversion and fixes running the http_ipv6 and quic tests on big endian architectures. --- src/include/ndpi_define.h.in | 22 ++++++++++++++++++++++ src/lib/protocols/quic.c | 4 ++-- 2 files changed, 24 insertions(+), 2 deletions(-) (limited to 'src/lib') diff --git a/src/include/ndpi_define.h.in b/src/include/ndpi_define.h.in index a73e03bc5..dc5fea74a 100644 --- a/src/include/ndpi_define.h.in +++ b/src/include/ndpi_define.h.in @@ -338,4 +338,26 @@ #define NDPI_MINOR @NDPI_MINOR@ #define NDPI_PATCH @NDPI_PATCH@ + +#ifdef __APPLE__ + +#include + +#define htobe16(x) OSSwapHostToBigInt16(x) +#define htole16(x) OSSwapHostToLittleInt16(x) +#define be16toh(x) OSSwapBigToHostInt16(x) +#define le16toh(x) OSSwapLittleToHostInt16(x) + +#define htobe32(x) OSSwapHostToBigInt32(x) +#define htole32(x) OSSwapHostToLittleInt32(x) +#define be32toh(x) OSSwapBigToHostInt32(x) +#define le32toh(x) OSSwapLittleToHostInt32(x) + +#define htobe64(x) OSSwapHostToBigInt64(x) +#define htole64(x) OSSwapHostToLittleInt64(x) +#define be64toh(x) OSSwapBigToHostInt64(x) +#define le64toh(x) OSSwapLittleToHostInt64(x) + +#endif /* __APPLE__ */ + #endif /* __NDPI_DEFINE_INCLUDE_FILE__ */ diff --git a/src/lib/protocols/quic.c b/src/lib/protocols/quic.c index e28db634a..322eb9be7 100644 --- a/src/lib/protocols/quic.c +++ b/src/lib/protocols/quic.c @@ -96,8 +96,8 @@ void ndpi_search_quic(struct ndpi_detection_module_struct *ndpi_struct, && (packet->payload[i+1] == 'N') && (packet->payload[i+2] == 'I') && (packet->payload[i+3] == 0)) { - u_int32_t offset = *((u_int32_t*)&packet->payload[i+4]); - u_int32_t prev_offset = *((u_int32_t*)&packet->payload[i-4]); + u_int32_t offset = le32toh(*((u_int32_t*)&packet->payload[i+4])); + u_int32_t prev_offset = le32toh(*((u_int32_t*)&packet->payload[i-4])); int len = offset-prev_offset; int sni_offset = i+prev_offset+1; -- cgit v1.2.3 From 2c672a4ea2a79fc1dad555c55a59b807fdfdd65b Mon Sep 17 00:00:00 2001 From: Ludovico Cavedon Date: Wed, 16 Jan 2019 01:15:43 -0800 Subject: Compute packet_direction consistently independently from endianness Ensure packet_direction is computed in the same way on little endian and big endian architectures. This change will convert IP addresses and port from little endian to host endian (instead of converting from network endian to host endian) so that it does not change the behavior on little endian architecture where ndpi is usually developed. A better (but more invasive) change would be to use the ntoh*() functions and then adjust all affected tests. --- src/lib/ndpi_main.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'src/lib') diff --git a/src/lib/ndpi_main.c b/src/lib/ndpi_main.c index 863943c4a..ac8543648 100644 --- a/src/lib/ndpi_main.c +++ b/src/lib/ndpi_main.c @@ -3693,7 +3693,7 @@ void ndpi_connection_tracking(struct ndpi_detection_module_struct *ndpi_struct, if(ndpi_struct->direction_detect_disable) { packet->packet_direction = flow->packet_direction; } else { - if(iph != NULL && iph->saddr < iph->daddr) + if(iph != NULL && le32toh(iph->saddr) < le32toh(iph->daddr)) packet->packet_direction = 1; #ifdef NDPI_DETECTION_SUPPORT_IPV6 @@ -3717,7 +3717,7 @@ void ndpi_connection_tracking(struct ndpi_detection_module_struct *ndpi_struct, packet->num_retried_bytes = 0; if(!ndpi_struct->direction_detect_disable) - packet->packet_direction = (tcph->source < tcph->dest) ? 1 : 0; + packet->packet_direction = (le16toh(tcph->source) < le16toh(tcph->dest)) ? 1 : 0; if(tcph->syn != 0 && tcph->ack == 0 && flow->l4.tcp.seen_syn == 0 && flow->l4.tcp.seen_syn_ack == 0 && flow->l4.tcp.seen_ack == 0) { @@ -3781,7 +3781,7 @@ void ndpi_connection_tracking(struct ndpi_detection_module_struct *ndpi_struct, } } else if(udph != NULL) { if(!ndpi_struct->direction_detect_disable) - packet->packet_direction = (udph->source < udph->dest) ? 1 : 0; + packet->packet_direction = (le16toh(udph->source) < le16toh(udph->dest)) ? 1 : 0; } if(flow->packet_counter < MAX_PACKET_COUNTER && packet->payload_packet_len) { -- cgit v1.2.3 From c856e0a56384ae9d145282d0eb6f8b97f54913fb Mon Sep 17 00:00:00 2001 From: Luca Deri Date: Wed, 16 Jan 2019 22:30:55 +0100 Subject: Disabled ookla statistics: needs to be improved --- src/lib/protocols/ookla.c | 2 ++ 1 file changed, 2 insertions(+) (limited to 'src/lib') diff --git a/src/lib/protocols/ookla.c b/src/lib/protocols/ookla.c index b1eb295a7..4d46dbf58 100644 --- a/src/lib/protocols/ookla.c +++ b/src/lib/protocols/ookla.c @@ -26,6 +26,7 @@ void ndpi_search_ookla(struct ndpi_detection_module_struct* ndpi_struct, struct ndpi_flow_struct* flow) { +#if 0 struct ndpi_packet_struct* packet = &flow->packet; u_int32_t addr = 0; void *value; @@ -49,6 +50,7 @@ void ndpi_search_ookla(struct ndpi_detection_module_struct* ndpi_struct, struct } ookla_exclude: +#endif NDPI_EXCLUDE_PROTO(ndpi_struct, flow); } -- cgit v1.2.3 From f3b0878a37d252f0a9300eab7a7df28d64146211 Mon Sep 17 00:00:00 2001 From: Luca Deri Date: Thu, 17 Jan 2019 00:39:45 +0100 Subject: Warnign fixes Updated test results --- example/Makefile.in | 3 +++ example/ndpiReader.c | 3 +-- example/ndpi_util.c | 2 +- src/lib/ndpi_main.c | 13 +++++++------ src/lib/protocols/coap.c | 2 +- src/lib/protocols/mdns_proto.c | 2 +- src/lib/protocols/ssl.c | 2 +- tests/result/pps.pcap.out | 7 +++---- tests/result/starcraft_battle.pcap.out | 5 ++--- tests/result/waze.pcap.out | 5 ++--- 10 files changed, 22 insertions(+), 22 deletions(-) (limited to 'src/lib') diff --git a/example/Makefile.in b/example/Makefile.in index bf218f503..074718028 100644 --- a/example/Makefile.in +++ b/example/Makefile.in @@ -22,6 +22,9 @@ install: dpdk: make -f Makefile.dpdk +check: + cppcheck --template='{file}:{line}:{severity}:{message}' --quiet --enable=all --force -I../src/include -I/usr/local/include/json-c *.c + clean: /bin/rm -f *.o ndpiReader ndpiReader.dpdk /bin/rm -f .*.dpdk.cmd .*.o.cmd *.dpdk.map .*.o.d diff --git a/example/ndpiReader.c b/example/ndpiReader.c index 6c3dfeee8..7fc468492 100644 --- a/example/ndpiReader.c +++ b/example/ndpiReader.c @@ -1015,7 +1015,6 @@ void freeIpTree(addr_node *root) { freeIpTree(root->left); freeIpTree(root->right); free(root); - root = NULL; } /* *********************************************** */ @@ -2901,7 +2900,7 @@ float getAverage(struct json_object *jObj_stat, char *field){ float average; float sum = 0; int r; - int j; + int j = 0; if((r = strcmp(field, "top.scanner.stats")) == 0) { for(j=0; j> 1; + r = ((r & 1) ? 0 : (uint32_t)0xEDB88320L) ^ r >> 1; return r ^ (uint32_t)0xFF000000L; } diff --git a/src/lib/ndpi_main.c b/src/lib/ndpi_main.c index 863943c4a..ad7c788ac 100644 --- a/src/lib/ndpi_main.c +++ b/src/lib/ndpi_main.c @@ -842,21 +842,21 @@ static int init_hyperscan(struct ndpi_detection_module_struct *ndpi_mod) { } need_to_be_free = (unsigned char*)calloc(sizeof(unsigned char), num_patterns + 1); - if (!need_to_be_free) { + if(!need_to_be_free) { free(expressions); free(ids); return(-1); } - for (i = 0, j = 0; host_match[i].string_to_match != NULL || host_match[i].pattern_to_match != NULL; i++) { - if (host_match[i].pattern_to_match) { + for(i = 0, j = 0; host_match[i].string_to_match != NULL || host_match[i].pattern_to_match != NULL; i++) { + if(host_match[i].pattern_to_match) { expressions[j] = host_match[i].pattern_to_match; ids[j] = host_match[i].protocol_id; need_to_be_free[j] = 0; ++j; } else { expressions[j] = string2hex(host_match[i].string_to_match); - if (expressions[j] != NULL) { + if(expressions[j] != NULL) { ids[j] = host_match[i].protocol_id; need_to_be_free[j] = 1; ++j; @@ -871,11 +871,12 @@ static int init_hyperscan(struct ndpi_detection_module_struct *ndpi_mod) { rc = hyperscan_load_patterns(hs, j, (const char**)expressions, ids); - for (i = 0; i < j; ++i) - if (need_to_be_free[i]) + for(i = 0; i < j; ++i) + if(need_to_be_free[i]) free(expressions[i]); free(expressions), free(ids); + free(need_to_be_free); return(rc); } diff --git a/src/lib/protocols/coap.c b/src/lib/protocols/coap.c index 5ac8cb80e..cf5061bbe 100644 --- a/src/lib/protocols/coap.c +++ b/src/lib/protocols/coap.c @@ -129,7 +129,7 @@ void ndpi_search_coap (struct ndpi_detection_module_struct *ndpi_struct, // check values in header if(h->version == 1) { if(h->type == CON || h->type == NO_CON || h->type == ACK || h->type == RST ) { - if(h->tkl == 0 || h->tkl < 8) { + if(h->tkl < 8) { if((h->code >= 0 && h->code <= 5) || (h->code >= 65 && h->code <= 69) || (h->code >= 128 && h->code <= 134) || (h->code >= 140 && h->code <= 143) || (h->code >= 160 && h->code <= 165)) { diff --git a/src/lib/protocols/mdns_proto.c b/src/lib/protocols/mdns_proto.c index 77bdf4208..00c7c8748 100644 --- a/src/lib/protocols/mdns_proto.c +++ b/src/lib/protocols/mdns_proto.c @@ -77,7 +77,7 @@ static int ndpi_int_check_mdns_payload(struct ndpi_detection_module_struct char answer[256]; int i, j, len; - for(i=13, j=0; (packet->payload[i] != 0) && (i < packet->payload_packet_len) && (i < (sizeof(answer)-1)); i++) + for(i=13, j=0; (i < packet->payload_packet_len) && (i < (sizeof(answer)-1)) && (packet->payload[i] != 0); i++) answer[j++] = (packet->payload[i] < 13) ? '.' : packet->payload[i]; answer[j] = '\0'; diff --git a/src/lib/protocols/ssl.c b/src/lib/protocols/ssl.c index 25d535a57..4651b358f 100644 --- a/src/lib/protocols/ssl.c +++ b/src/lib/protocols/ssl.c @@ -559,7 +559,7 @@ static u_int8_t ndpi_search_sslv3_direction1(struct ndpi_detection_module_struct } } - if((packet->payload_packet_len > temp && packet->payload_packet_len > 100) && packet->payload_packet_len > 9) { + if((packet->payload_packet_len > temp) && (packet->payload_packet_len > 100)) { /* the server hello may be split into small packets and the certificate has its own SSL Record * so temp contains only the length for the first ServerHello block */ u_int32_t cert_start; diff --git a/tests/result/pps.pcap.out b/tests/result/pps.pcap.out index 778aa1197..f9e26850b 100644 --- a/tests/result/pps.pcap.out +++ b/tests/result/pps.pcap.out @@ -1,7 +1,6 @@ Unknown 990 378832 34 -HTTP 47 42014 11 +HTTP 73 69236 13 SSDP 63 17143 10 -HTTP_Download 26 27222 2 Google 2 1093 1 GenericProtocol 1429 1780307 49 @@ -9,7 +8,7 @@ GenericProtocol 1429 1780307 49 2 TCP 192.168.115.8:50778 <-> 223.26.106.20:80 [proto: 7.137/HTTP.GenericProtocol][cat: Streaming/17][1 pkts/303 bytes <-> 528 pkts/692658 bytes][Host: preimage1.qiyipic.com] 3 TCP 192.168.115.8:50505 <-> 223.26.106.19:80 [proto: 7.137/HTTP.GenericProtocol][cat: Streaming/17][2 pkts/400 bytes <-> 244 pkts/319633 bytes][Host: static.qiyi.com] 4 TCP 192.168.115.8:50491 <-> 223.26.106.66:80 [proto: 7/HTTP][cat: Web/5][1 pkts/426 bytes <-> 26 pkts/33872 bytes][Host: 223.26.106.66] - 5 TCP 192.168.115.8:50486 <-> 77.234.40.96:80 [proto: 7.60/HTTP.HTTP_Download][cat: Download-FileTransfer-FileSharing/7][11 pkts/11023 bytes <-> 12 pkts/14869 bytes][Host: bcu.ff.avast.com] + 5 TCP 192.168.115.8:50486 <-> 77.234.40.96:80 [proto: 7/HTTP][cat: Web/5][11 pkts/11023 bytes <-> 12 pkts/14869 bytes][Host: bcu.ff.avast.com] 6 UDP 192.168.5.38:1900 -> 239.255.255.250:1900 [proto: 12/SSDP][cat: System/18][18 pkts/9327 bytes -> 0 pkts/0 bytes] 7 TCP 192.168.115.8:50476 <-> 101.227.32.39:80 [proto: 7.137/HTTP.GenericProtocol][cat: Streaming/17][1 pkts/656 bytes <-> 4 pkts/3897 bytes][Host: cache.video.iqiyi.com] 8 TCP 192.168.115.8:50495 <-> 202.108.14.236:80 [proto: 7.137/HTTP.GenericProtocol][cat: Streaming/17][3 pkts/2844 bytes <-> 3 pkts/597 bytes][Host: msg.71.am] @@ -26,7 +25,7 @@ GenericProtocol 1429 1780307 49 19 TCP 192.168.115.8:50464 <-> 123.125.112.49:80 [proto: 7.137/HTTP.GenericProtocol][cat: Streaming/17][1 pkts/707 bytes <-> 1 pkts/744 bytes][Host: click.hm.baidu.com] 20 TCP 192.168.115.8:50492 <-> 111.206.13.3:80 [proto: 7.137/HTTP.GenericProtocol][cat: Streaming/17][1 pkts/389 bytes <-> 2 pkts/1034 bytes][Host: pdata.video.qiyi.com] 21 TCP 192.168.115.8:50777 <-> 111.206.22.77:80 [proto: 7.137/HTTP.GenericProtocol][cat: Streaming/17][1 pkts/1186 bytes <-> 1 pkts/194 bytes][Host: msg.iqiyi.com] - 22 TCP 192.168.115.8:50494 <-> 223.26.106.66:80 [proto: 7.60/HTTP.HTTP_Download][cat: Download-FileTransfer-FileSharing/7][2 pkts/887 bytes <-> 1 pkts/443 bytes][Host: 223.26.106.66] + 22 TCP 192.168.115.8:50494 <-> 223.26.106.66:80 [proto: 7/HTTP][cat: Web/5][2 pkts/887 bytes <-> 1 pkts/443 bytes][Host: 223.26.106.66] 23 TCP 192.168.115.8:50497 <-> 123.125.112.49:80 [proto: 7.137/HTTP.GenericProtocol][cat: Streaming/17][1 pkts/1004 bytes <-> 2 pkts/301 bytes][Host: click.hm.baidu.com] 24 TCP 192.168.115.8:50499 <-> 111.206.22.76:80 [proto: 7.137/HTTP.GenericProtocol][cat: Streaming/17][1 pkts/1097 bytes <-> 1 pkts/199 bytes][Host: msg.iqiyi.com] 25 TCP 192.168.115.8:50474 <-> 202.108.14.221:80 [proto: 7.137/HTTP.GenericProtocol][cat: Streaming/17][1 pkts/1100 bytes <-> 1 pkts/194 bytes][Host: msg.iqiyi.com] diff --git a/tests/result/starcraft_battle.pcap.out b/tests/result/starcraft_battle.pcap.out index c63b86dc8..8d42d972d 100644 --- a/tests/result/starcraft_battle.pcap.out +++ b/tests/result/starcraft_battle.pcap.out @@ -1,7 +1,6 @@ DNS 26 2848 7 -HTTP 271 160676 18 +HTTP 450 294880 19 SSDP 11 4984 1 -HTTP_Download 179 134204 1 WorldOfWarcraft 9 880 1 IGMP 2 120 1 SSL 38 2548 11 @@ -9,7 +8,7 @@ Google 22 2184 5 Github 3 234 1 Starcraft 236 51494 6 - 1 TCP 192.168.1.100:3508 <-> 87.248.221.254:80 [proto: 7.60/HTTP.HTTP_Download][cat: Download-FileTransfer-FileSharing/7][90 pkts/5059 bytes <-> 89 pkts/129145 bytes][Host: llnw.blizzard.com] + 1 TCP 192.168.1.100:3508 <-> 87.248.221.254:80 [proto: 7/HTTP][cat: Web/5][90 pkts/5059 bytes <-> 89 pkts/129145 bytes][Host: llnw.blizzard.com] 2 TCP 192.168.1.100:3517 <-> 213.248.127.130:1119 [proto: 213/Starcraft][cat: Game/8][126 pkts/9157 bytes <-> 89 pkts/41021 bytes] 3 TCP 192.168.1.100:3527 <-> 2.228.46.112:80 [proto: 7/HTTP][cat: Web/5][15 pkts/971 bytes <-> 26 pkts/36462 bytes][Host: bnetcmsus-a.akamaihd.net] 4 TCP 192.168.1.100:3528 <-> 2.228.46.112:80 [proto: 7/HTTP][cat: Web/5][11 pkts/755 bytes <-> 18 pkts/24350 bytes][Host: bnetcmsus-a.akamaihd.net] diff --git a/tests/result/waze.pcap.out b/tests/result/waze.pcap.out index 5bdf0a3bb..c321981a5 100644 --- a/tests/result/waze.pcap.out +++ b/tests/result/waze.pcap.out @@ -1,14 +1,13 @@ Unknown 10 786 1 -HTTP 28 1572 7 +HTTP 65 64777 8 NTP 2 180 1 -HTTP_Download 37 63205 1 SSL 8 432 2 Google 13 2142 1 Waze 484 289335 19 WhatsApp 15 1341 1 1 TCP 10.8.0.1:36100 <-> 46.51.173.182:443 [proto: 91.135/SSL.Waze][cat: Web/5][52 pkts/10860 bytes <-> 55 pkts/74852 bytes][server: *.world.waze.com] - 2 TCP 10.8.0.1:54915 <-> 65.39.128.135:80 [proto: 7.60/HTTP.HTTP_Download][cat: Download-FileTransfer-FileSharing/7][19 pkts/1309 bytes <-> 18 pkts/61896 bytes][Host: xtra1.gpsonextra.net] + 2 TCP 10.8.0.1:54915 <-> 65.39.128.135:80 [proto: 7/HTTP][cat: Web/5][19 pkts/1309 bytes <-> 18 pkts/61896 bytes][Host: xtra1.gpsonextra.net] 3 TCP 10.8.0.1:39021 <-> 52.17.114.219:443 [proto: 91.135/SSL.Waze][cat: Web/5][17 pkts/1962 bytes <-> 16 pkts/56934 bytes][server: *.world.waze.com] 4 TCP 10.8.0.1:36312 <-> 176.34.186.180:443 [proto: 91.135/SSL.Waze][cat: Web/5][17 pkts/2176 bytes <-> 15 pkts/42443 bytes][server: *.world.waze.com] 5 TCP 10.8.0.1:36316 <-> 176.34.186.180:443 [proto: 91.135/SSL.Waze][cat: Web/5][15 pkts/1540 bytes <-> 13 pkts/26346 bytes][server: *.world.waze.com] -- cgit v1.2.3 From efef99cbadc8ddd6f6743e04d184fe240d6eb334 Mon Sep 17 00:00:00 2001 From: Luca Deri Date: Thu, 17 Jan 2019 00:40:28 +0100 Subject: Removed this party LRU and replaced with home grown --- example/ndpiReader.c | 5 +- src/include/Makefile.am | 2 +- src/include/ndpi_api.h | 6 + src/include/ndpi_typedefs.h | 6 +- src/lib/ndpi_main.c | 48 +++++- src/lib/protocols/http.c | 19 +-- src/lib/protocols/ookla.c | 17 +-- src/lib/protocols/ssl.c | 4 + src/lib/third_party/include/lruc.h | 55 ------- src/lib/third_party/src/lruc.c | 294 ------------------------------------- 10 files changed, 79 insertions(+), 377 deletions(-) delete mode 100644 src/lib/third_party/include/lruc.h delete mode 100644 src/lib/third_party/src/lruc.c (limited to 'src/lib') diff --git a/example/ndpiReader.c b/example/ndpiReader.c index 6c3dfeee8..8996ea1a5 100644 --- a/example/ndpiReader.c +++ b/example/ndpiReader.c @@ -761,6 +761,9 @@ static void printFlow(u_int16_t id, struct ndpi_flow_info *flow, u_int16_t threa if((verbose != 1) && (verbose != 2)) return; + if(5222 == ntohs(flow->dst_port)) + printf("************\n"); + if(!json_flag) { fprintf(out, "\t%u", id); @@ -913,7 +916,7 @@ static void node_proto_guess_walker(const void *node, ndpi_VISIT which, int dept struct ndpi_flow_info *flow = *(struct ndpi_flow_info **) node; u_int16_t thread_id = *((u_int16_t *) user_data); - if((which == ndpi_preorder) || (which == ndpi_leaf)) { /* Avoid walking the same node multiple times */ + if((which == ndpi_preorder) || (which == ndpi_leaf)) { /* Avoid walking the same node multiple times */ if((!flow->detection_completed) && flow->ndpi_flow) flow->detected_protocol = ndpi_detection_giveup(ndpi_thread_info[0].workflow->ndpi_struct, flow->ndpi_flow, enable_protocol_guess); diff --git a/src/include/Makefile.am b/src/include/Makefile.am index 6eeca93b4..47fcbd224 100644 --- a/src/include/Makefile.am +++ b/src/include/Makefile.am @@ -1,4 +1,4 @@ -library_includedir=$(includedir)/libndpi-@VERSION@/libndpi +plibrary_includedir=$(includedir)/libndpi-@VERSION@/libndpi library_include_HEADERS = ndpi_api.h \ ndpi_define.h \ diff --git a/src/include/ndpi_api.h b/src/include/ndpi_api.h index e57f3a568..e09c91c8e 100644 --- a/src/include/ndpi_api.h +++ b/src/include/ndpi_api.h @@ -756,6 +756,12 @@ extern "C" { u_int ndpi_get_ndpi_num_custom_protocols(struct ndpi_detection_module_struct *ndpi_mod); u_int ndpi_get_ndpi_detection_module_size(); void ndpi_set_log_level(struct ndpi_detection_module_struct *ndpi_mod, u_int l); + + /* LRU cache */ + struct ndpi_lru_cache* ndpi_lru_cache_init(u_int32_t num_entries); + void ndpi_lru_free_cache(struct ndpi_lru_cache *c); + u_int8_t ndpi_lru_find_cache(struct ndpi_lru_cache *c, u_int32_t key, u_int8_t clean_key_when_found); + void ndpi_lru_add_to_cache(struct ndpi_lru_cache *c, u_int32_t key); /** * Add a string to match to an automata diff --git a/src/include/ndpi_typedefs.h b/src/include/ndpi_typedefs.h index 34d308e2e..cf1af8bc3 100644 --- a/src/include/ndpi_typedefs.h +++ b/src/include/ndpi_typedefs.h @@ -367,6 +367,10 @@ typedef enum { HTTP_METHOD_CONNECT } ndpi_http_method; +struct ndpi_lru_cache { + u_int32_t num_entries, *entries; +}; + struct ndpi_id_struct { /** detected_protocol_bitmask: @@ -997,7 +1001,7 @@ struct ndpi_detection_module_struct { int bt_ann_len; /* NDPI_PROTOCOL_OOKLA */ - void *ookla_cache; + struct ndpi_lru_cache *ookla_cache; /* NDPI_PROTOCOL_TINC */ struct cache *tinc_cache; diff --git a/src/lib/ndpi_main.c b/src/lib/ndpi_main.c index 863943c4a..594d521d3 100644 --- a/src/lib/ndpi_main.c +++ b/src/lib/ndpi_main.c @@ -30,7 +30,6 @@ #include #include "ahocorasick.h" #include "libcache.h" -#include "lruc.h" #define NDPI_CURRENT_PROTO NDPI_PROTOCOL_UNKNOWN @@ -2433,7 +2432,7 @@ void ndpi_exit_detection_module(struct ndpi_detection_module_struct *ndpi_struct cache_free((cache_t)(ndpi_struct->tinc_cache)); if(ndpi_struct->ookla_cache) - lruc_free((lruc*)ndpi_struct->ookla_cache); + ndpi_lru_free_cache(ndpi_struct->ookla_cache); if(ndpi_struct->protocols_ptree) ndpi_Destroy_Patricia((patricia_tree_t*)ndpi_struct->protocols_ptree, free_ptree_data); @@ -3995,6 +3994,9 @@ ndpi_protocol ndpi_detection_giveup(struct ndpi_detection_module_struct *ndpi_st if(flow == NULL) return(ret); + if(flow->packet.tcp && (5222 == ntohs(flow->packet.tcp->dest))) + printf("%u - %u\n", ntohs(flow->packet.tcp->source), ntohs(flow->packet.tcp->dest)); + /* TODO: add the remaining stage_XXXX protocols */ if(flow->detected_protocol_stack[0] == NDPI_PROTOCOL_UNKNOWN) { u_int16_t guessed_protocol_id, guessed_host_protocol_id; @@ -6171,6 +6173,48 @@ void ndpi_set_log_level(struct ndpi_detection_module_struct *ndpi_mod, u_int l) /* ******************************************************************** */ +/* LRU cache */ + +struct ndpi_lru_cache* ndpi_lru_cache_init(u_int32_t num_entries) { + struct ndpi_lru_cache *c = (struct ndpi_lru_cache*)malloc(sizeof(struct ndpi_lru_cache)); + + if(!c) return(NULL); + + c->entries = (u_int32_t*)calloc(num_entries, sizeof(u_int32_t)); + + if(!c->entries) { + free(c); + return(NULL); + } else + c->num_entries = num_entries; + + return(c); +} + +void ndpi_lru_free_cache(struct ndpi_lru_cache *c) { + free(c->entries); + free(c); +} + + +u_int8_t ndpi_lru_find_cache(struct ndpi_lru_cache *c, u_int32_t key, u_int8_t clean_key_when_found) { + u_int32_t slot = key % c->num_entries; + + if(c->entries[slot] == key) { + if(clean_key_when_found) c->entries[slot] = 0; + return(1); + } else + return(0); +} + +void ndpi_lru_add_to_cache(struct ndpi_lru_cache *c, u_int32_t key) { + u_int32_t slot = key % c->num_entries; + + c->entries[slot] = key; +} + +/* ******************************************************************** */ + /* NOTE: - Leave fields empty/zero when information is missing (e.g. with ICMP ports are zero) diff --git a/src/lib/protocols/http.c b/src/lib/protocols/http.c index 661e55732..16b122d06 100644 --- a/src/lib/protocols/http.c +++ b/src/lib/protocols/http.c @@ -26,7 +26,6 @@ #define NDPI_CURRENT_PROTO NDPI_PROTOCOL_HTTP #include "ndpi_api.h" -#include "lruc.h" static void ndpi_int_http_add_connection(struct ndpi_detection_module_struct *ndpi_struct, struct ndpi_flow_struct *flow, @@ -642,19 +641,15 @@ static void ndpi_check_http_tcp(struct ndpi_detection_module_struct *ndpi_struct ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_OOKLA, NDPI_PROTOCOL_UNKNOWN); if(ndpi_struct->ookla_cache == NULL) - ndpi_struct->ookla_cache = lruc_new(4*1024, 1024); - + ndpi_struct->ookla_cache = ndpi_lru_cache_init(1024); + if(packet->iph != NULL && ndpi_struct->ookla_cache != NULL) { - u_int8_t *dummy = (u_int8_t*)ndpi_malloc(sizeof(u_int8_t)); - - if(dummy) { - if(packet->tcp->source == htons(8080)) - lruc_set((lruc*)ndpi_struct->ookla_cache, (void*)&packet->iph->saddr, 4, dummy, 1); - else - lruc_set((lruc*)ndpi_struct->ookla_cache, (void*)&packet->iph->daddr, 4, dummy, 1); - } + if(packet->tcp->source == htons(8080)) + ndpi_lru_add_to_cache(ndpi_struct->ookla_cache, packet->iph->saddr); + else + ndpi_lru_add_to_cache(ndpi_struct->ookla_cache, packet->iph->daddr); } - + return; } diff --git a/src/lib/protocols/ookla.c b/src/lib/protocols/ookla.c index 4d46dbf58..06d97e216 100644 --- a/src/lib/protocols/ookla.c +++ b/src/lib/protocols/ookla.c @@ -22,36 +22,32 @@ #define NDPI_CURRENT_PROTO NDPI_PROTOCOL_OOKLA #include "ndpi_api.h" -#include "lruc.h" void ndpi_search_ookla(struct ndpi_detection_module_struct* ndpi_struct, struct ndpi_flow_struct* flow) { -#if 0 struct ndpi_packet_struct* packet = &flow->packet; u_int32_t addr = 0; void *value; - + NDPI_LOG_DBG(ndpi_struct, "Ookla detection\n"); - + if(packet->tcp->source == htons(8080)) addr = packet->iph->saddr; else if(packet->tcp->dest == htons(8080)) addr = packet->iph->daddr; else goto ookla_exclude; - + if(ndpi_struct->ookla_cache != NULL) { - if(lruc_get(ndpi_struct->ookla_cache, &addr, sizeof(addr), &value) == LRUC_NO_ERROR) { - /* Don't remove it as it can be used for other connections */ + if(ndpi_lru_find_cache(ndpi_struct->ookla_cache, addr, 0 /* Don't remove it as it can be used for other connections */)) { NDPI_LOG_INFO(ndpi_struct, "found ookla tcp connection\n"); ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_OOKLA, NDPI_PROTOCOL_UNKNOWN); return; - } + } } ookla_exclude: -#endif - NDPI_EXCLUDE_PROTO(ndpi_struct, flow); + NDPI_EXCLUDE_PROTO(ndpi_struct, flow); } void init_ookla_dissector(struct ndpi_detection_module_struct *ndpi_struct, @@ -65,4 +61,3 @@ void init_ookla_dissector(struct ndpi_detection_module_struct *ndpi_struct, *id += 1; } - diff --git a/src/lib/protocols/ssl.c b/src/lib/protocols/ssl.c index 25d535a57..785ffa70d 100644 --- a/src/lib/protocols/ssl.c +++ b/src/lib/protocols/ssl.c @@ -626,6 +626,10 @@ void ndpi_search_ssl_tcp(struct ndpi_detection_module_struct *ndpi_struct, struc struct ndpi_packet_struct *packet = &flow->packet; u_int8_t ret; + if(flow->packet.tcp && (5222 == ntohs(flow->packet.tcp->dest))) + printf("%u - %u\n", ntohs(flow->packet.tcp->source), ntohs(flow->packet.tcp->dest)); + + if(packet->detected_protocol_stack[0] == NDPI_PROTOCOL_SSL) { if(flow->l4.tcp.ssl_stage == 3 && packet->payload_packet_len > 20 && flow->packet_counter < 5) { /* this should only happen, when we detected SSL with a packet that had parts of the certificate in subsequent packets diff --git a/src/lib/third_party/include/lruc.h b/src/lib/third_party/include/lruc.h deleted file mode 100644 index 55fb271fe..000000000 --- a/src/lib/third_party/include/lruc.h +++ /dev/null @@ -1,55 +0,0 @@ -#include -#include -#include - -#ifndef __lruc_header__ -#define __lruc_header__ - -// ------------------------------------------ -// errors -// ------------------------------------------ -typedef enum { - LRUC_NO_ERROR = 0, - LRUC_MISSING_CACHE, - LRUC_MISSING_KEY, - LRUC_MISSING_VALUE, - LRUC_PTHREAD_ERROR, - LRUC_VALUE_TOO_LARGE -} lruc_error; - - -// ------------------------------------------ -// types -// ------------------------------------------ -typedef struct { - void *value; - void *key; - uint32_t value_length; - uint32_t key_length; - uint64_t access_count; - void *next; -} lruc_item; - -typedef struct { - lruc_item **items; - uint64_t access_count; - uint64_t free_memory; - uint64_t total_memory; - uint64_t average_item_length; - uint32_t hash_table_size; - time_t seed; - lruc_item *free_items; - pthread_mutex_t *mutex; -} lruc; - - -// ------------------------------------------ -// api -// ------------------------------------------ -lruc *lruc_new(uint64_t cache_size, uint32_t average_length); -lruc_error lruc_free(lruc *cache); -lruc_error lruc_set(lruc *cache, void *key, uint32_t key_length, void *value, uint32_t value_length); -lruc_error lruc_get(lruc *cache, void *key, uint32_t key_length, void **value); -lruc_error lruc_delete(lruc *cache, void *key, uint32_t key_length); - -#endif diff --git a/src/lib/third_party/src/lruc.c b/src/lib/third_party/src/lruc.c deleted file mode 100644 index f08fb2ce1..000000000 --- a/src/lib/third_party/src/lruc.c +++ /dev/null @@ -1,294 +0,0 @@ -/* https://github.com/willcannings/C-LRU-Cache */ - -#include "lruc.h" -#include -#include -#include -#include - -// ------------------------------------------ -// private functions -// ------------------------------------------ -// MurmurHash2, by Austin Appleby -// http://sites.google.com/site/murmurhash/ -uint32_t lruc_hash(lruc *cache, void *key, uint32_t key_length) { - uint32_t m = 0x5bd1e995; - uint32_t r = 24; - uint32_t h = cache->seed ^ key_length; - char* data = (char *)key; - - while(key_length >= 4) { - uint32_t k = *(uint32_t *)data; - k *= m; - k ^= k >> r; - k *= m; - h *= m; - h ^= k; - data += 4; - key_length -= 4; - } - - switch(key_length) { - case 3: h ^= data[2] << 16; - case 2: h ^= data[1] << 8; - case 1: h ^= data[0]; - h *= m; - }; - - h ^= h >> 13; - h *= m; - h ^= h >> 15; - return h % cache->hash_table_size; -} - -// compare a key against an existing item's key -int lruc_cmp_keys(lruc_item *item, void *key, uint32_t key_length) { - if(key_length != item->key_length) - return 1; - else - return memcmp(key, item->key, key_length); -} - -// remove an item and push it to the free items queue -void lruc_remove_item(lruc *cache, lruc_item *prev, lruc_item *item, uint32_t hash_index) { - if(prev) - prev->next = item->next; - else - cache->items[hash_index] = (lruc_item *) item->next; - - // free memory and update the free memory counter - cache->free_memory += item->value_length; - free(item->value); - free(item->key); - - // push the item to the free items queue - memset(item, 0, sizeof(lruc_item)); - item->next = cache->free_items; - cache->free_items = item; -} - -// remove the least recently used item -// TODO: we can optimise this by finding the n lru items, where n = required_space / average_length -void lruc_remove_lru_item(lruc *cache) { - lruc_item *min_item = NULL, *min_prev = NULL; - lruc_item *item = NULL, *prev = NULL; - uint32_t i = 0, min_index = -1; - uint64_t min_access_count = -1; - - for(; i < cache->hash_table_size; i++) { - item = cache->items[i]; - prev = NULL; - - while(item) { - if(item->access_count < min_access_count || min_access_count == -1) { - min_access_count = item->access_count; - min_item = item; - min_prev = prev; - min_index = i; - } - prev = item; - item = item->next; - } - } - - if(min_item) - lruc_remove_item(cache, min_prev, min_item, min_index); -} - -// pop an existing item off the free queue, or create a new one -lruc_item *lruc_pop_or_create_item(lruc *cache) { - lruc_item *item = NULL; - - if(cache->free_items) { - item = cache->free_items; - cache->free_items = item->next; - } else { - item = (lruc_item *) calloc(sizeof(lruc_item), 1); - } - - return item; -} - -// error helpers -#define error_for(conditions, error) if(conditions) {return error;} -#define test_for_missing_cache() error_for(!cache, LRUC_MISSING_CACHE) -#define test_for_missing_key() error_for(!key || key_length == 0, LRUC_MISSING_KEY) -#define test_for_missing_value() error_for(!value || value_length == 0, LRUC_MISSING_VALUE) -#define test_for_value_too_large() error_for(value_length > cache->total_memory, LRUC_VALUE_TOO_LARGE) - -// lock helpers -#define lock_cache() if(pthread_mutex_lock(cache->mutex)) {\ - perror("LRU Cache unable to obtain mutex lock");\ - return LRUC_PTHREAD_ERROR;\ -} - -#define unlock_cache() if(pthread_mutex_unlock(cache->mutex)) {\ - perror("LRU Cache unable to release mutex lock");\ - return LRUC_PTHREAD_ERROR;\ -} - - -// ------------------------------------------ -// public api -// ------------------------------------------ -lruc *lruc_new(uint64_t cache_size, uint32_t average_length) { - // create the cache - lruc *cache = (lruc *) calloc(sizeof(lruc), 1); - if(!cache) { - perror("LRU Cache unable to create cache object"); - return NULL; - } - cache->hash_table_size = cache_size / average_length; - cache->average_item_length = average_length; - cache->free_memory = cache_size; - cache->total_memory = cache_size; - cache->seed = time(NULL); - - // size the hash table to a guestimate of the number of slots required (assuming a perfect hash) - cache->items = (lruc_item **) calloc(sizeof(lruc_item *), cache->hash_table_size); - if(!cache->items) { - perror("LRU Cache unable to create cache hash table"); - free(cache); - return NULL; - } - - // all cache calls are guarded by a mutex - cache->mutex = (pthread_mutex_t *) malloc(sizeof(pthread_mutex_t)); - if(pthread_mutex_init(cache->mutex, NULL)) { - perror("LRU Cache unable to initialise mutex"); - free(cache->items); - free(cache); - return NULL; - } - return cache; -} - - -lruc_error lruc_free(lruc *cache) { - test_for_missing_cache(); - - // free each of the cached items, and the hash table - lruc_item *item = NULL, *next = NULL; - uint32_t i = 0; - if(cache->items) { - for(; i < cache->hash_table_size; i++) { - item = cache->items[i]; - while(item) { - next = (lruc_item *) item->next; - free(item); - item = next; - } - } - free(cache->items); - } - - // free the cache - if(cache->mutex) { - if(pthread_mutex_destroy(cache->mutex)) { - perror("LRU Cache unable to destroy mutex"); - return LRUC_PTHREAD_ERROR; - } - } - free(cache); - - return LRUC_NO_ERROR; -} - - -lruc_error lruc_set(lruc *cache, void *key, uint32_t key_length, void *value, uint32_t value_length) { - test_for_missing_cache(); - test_for_missing_key(); - test_for_missing_value(); - test_for_value_too_large(); - lock_cache(); - - // see if the key already exists - uint32_t hash_index = lruc_hash(cache, key, key_length), required = 0; - lruc_item *item = NULL, *prev = NULL; - item = cache->items[hash_index]; - - while(item && lruc_cmp_keys(item, key, key_length)) { - prev = item; - item = (lruc_item *) item->next; - } - - if(item) { - // update the value and value_lengths - required = value_length - item->value_length; - free(item->value); - item->value = value; - item->value_length = value_length; - - } else { - // insert a new item - item = lruc_pop_or_create_item(cache); - item->value = value; - item->key = key; - item->value_length = value_length; - item->key_length = key_length; - required = value_length; - - if(prev) - prev->next = item; - else - cache->items[hash_index] = item; - } - item->access_count = ++cache->access_count; - - // remove as many items as necessary to free enough space - if(required > 0 && required > cache->free_memory) { - while(cache->free_memory < required) - lruc_remove_lru_item(cache); - } - cache->free_memory -= required; - unlock_cache(); - return LRUC_NO_ERROR; -} - - -lruc_error lruc_get(lruc *cache, void *key, uint32_t key_length, void **value) { - test_for_missing_cache(); - test_for_missing_key(); - lock_cache(); - - // loop until we find the item, or hit the end of a chain - uint32_t hash_index = lruc_hash(cache, key, key_length); - lruc_item *item = cache->items[hash_index]; - - while(item && lruc_cmp_keys(item, key, key_length)) - item = (lruc_item *) item->next; - - if(item) { - *value = item->value; - item->access_count = ++cache->access_count; - } else { - *value = NULL; - } - - unlock_cache(); - return LRUC_NO_ERROR; -} - - -lruc_error lruc_delete(lruc *cache, void *key, uint32_t key_length) { - test_for_missing_cache(); - test_for_missing_key(); - lock_cache(); - - // loop until we find the item, or hit the end of a chain - lruc_item *item = NULL, *prev = NULL; - uint32_t hash_index = lruc_hash(cache, key, key_length); - item = cache->items[hash_index]; - - while(item && lruc_cmp_keys(item, key, key_length)) { - prev = item; - item = (lruc_item *) item->next; - } - - if(item) { - lruc_remove_item(cache, prev, item, hash_index); - } - - unlock_cache(); - return LRUC_NO_ERROR; -} -- cgit v1.2.3 From 85155d589980192a9761a7fc3ef07b90491cad87 Mon Sep 17 00:00:00 2001 From: Luca Deri Date: Thu, 17 Jan 2019 00:42:45 +0100 Subject: Removed debug code --- example/ndpiReader.c | 3 --- src/lib/ndpi_main.c | 3 --- 2 files changed, 6 deletions(-) (limited to 'src/lib') diff --git a/example/ndpiReader.c b/example/ndpiReader.c index e30df41f9..3f030265c 100644 --- a/example/ndpiReader.c +++ b/example/ndpiReader.c @@ -761,9 +761,6 @@ static void printFlow(u_int16_t id, struct ndpi_flow_info *flow, u_int16_t threa if((verbose != 1) && (verbose != 2)) return; - if(5222 == ntohs(flow->dst_port)) - printf("************\n"); - if(!json_flag) { fprintf(out, "\t%u", id); diff --git a/src/lib/ndpi_main.c b/src/lib/ndpi_main.c index 23ea29947..8ea0de499 100644 --- a/src/lib/ndpi_main.c +++ b/src/lib/ndpi_main.c @@ -3995,9 +3995,6 @@ ndpi_protocol ndpi_detection_giveup(struct ndpi_detection_module_struct *ndpi_st if(flow == NULL) return(ret); - if(flow->packet.tcp && (5222 == ntohs(flow->packet.tcp->dest))) - printf("%u - %u\n", ntohs(flow->packet.tcp->source), ntohs(flow->packet.tcp->dest)); - /* TODO: add the remaining stage_XXXX protocols */ if(flow->detected_protocol_stack[0] == NDPI_PROTOCOL_UNKNOWN) { u_int16_t guessed_protocol_id, guessed_host_protocol_id; -- cgit v1.2.3 From 7efe1f06eedbdfbdc316d330f746c47391be8f78 Mon Sep 17 00:00:00 2001 From: Luca Deri Date: Thu, 17 Jan 2019 22:03:39 +0100 Subject: Removed debug code --- src/lib/protocols/ssl.c | 4 ---- 1 file changed, 4 deletions(-) (limited to 'src/lib') diff --git a/src/lib/protocols/ssl.c b/src/lib/protocols/ssl.c index ac31fb715..4651b358f 100644 --- a/src/lib/protocols/ssl.c +++ b/src/lib/protocols/ssl.c @@ -626,10 +626,6 @@ void ndpi_search_ssl_tcp(struct ndpi_detection_module_struct *ndpi_struct, struc struct ndpi_packet_struct *packet = &flow->packet; u_int8_t ret; - if(flow->packet.tcp && (5222 == ntohs(flow->packet.tcp->dest))) - printf("%u - %u\n", ntohs(flow->packet.tcp->source), ntohs(flow->packet.tcp->dest)); - - if(packet->detected_protocol_stack[0] == NDPI_PROTOCOL_SSL) { if(flow->l4.tcp.ssl_stage == 3 && packet->payload_packet_len > 20 && flow->packet_counter < 5) { /* this should only happen, when we detected SSL with a packet that had parts of the certificate in subsequent packets -- cgit v1.2.3 From 81649d9979d5069299500bc6d74fc9587cccee16 Mon Sep 17 00:00:00 2001 From: Pramode Date: Sun, 20 Jan 2019 19:09:07 +0530 Subject: Propagate prefix settings to Makefiles A "make install" was failing because the --prefix flag setting was not being propagated to the Makefiles. --- example/Makefile.in | 2 +- src/lib/Makefile.in | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'src/lib') diff --git a/example/Makefile.in b/example/Makefile.in index 074718028..8960ed0a2 100644 --- a/example/Makefile.in +++ b/example/Makefile.in @@ -3,7 +3,7 @@ CFLAGS=-g -I../src/include @CFLAGS@ LIBNDPI=../src/lib/libndpi.a LDFLAGS=$(LIBNDPI) @PCAP_LIB@ -lpthread @LDFLAGS@ OBJS=ndpiReader.o ndpi_util.o -PREFIX?=/usr/local +PREFIX?=@prefix@ all: ndpiReader @DPDK_TARGET@ diff --git a/src/lib/Makefile.in b/src/lib/Makefile.in index 0ffeb9db5..8b0853dd4 100644 --- a/src/lib/Makefile.in +++ b/src/lib/Makefile.in @@ -10,7 +10,7 @@ # # Installation directories # -prefix = /usr/local +prefix = @prefix@ libdir = ${prefix}/lib includedir = ${prefix}/include/ndpi CC = @CC@ -- cgit v1.2.3 From e2ffcd722958857df232d8a02a9809b05d861049 Mon Sep 17 00:00:00 2001 From: Luca Deri Date: Wed, 23 Jan 2019 00:53:56 +0100 Subject: Improved HTTP response code handling --- src/include/ndpi_typedefs.h | 2 +- src/lib/ndpi_main.c | 18 ++++++++++++------ src/lib/protocols/http.c | 22 +++++++++++++++++++--- 3 files changed, 32 insertions(+), 10 deletions(-) (limited to 'src/lib') diff --git a/src/include/ndpi_typedefs.h b/src/include/ndpi_typedefs.h index cf1af8bc3..172763789 100644 --- a/src/include/ndpi_typedefs.h +++ b/src/include/ndpi_typedefs.h @@ -1071,7 +1071,7 @@ struct ndpi_flow_struct { char *url, *content_type; u_int8_t num_request_headers, num_response_headers; u_int8_t request_version; /* 0=1.0 and 1=1.1. Create an enum for this? */ - u_char response_status_code[5]; /* 200, 404, etc. */ + u_int16_t response_status_code; /* 200, 404, etc. */ } http; union { diff --git a/src/lib/ndpi_main.c b/src/lib/ndpi_main.c index fcf86449a..0b7af360d 100644 --- a/src/lib/ndpi_main.c +++ b/src/lib/ndpi_main.c @@ -4792,12 +4792,18 @@ void ndpi_parse_packet_line_info(struct ndpi_detection_module_struct *ndpi_struc packet->http_num_headers++; /* Set server HTTP response code */ - strncpy((char*)flow->http.response_status_code, (char*)packet->http_response.ptr, 3); - flow->http.response_status_code[4] = '\0'; - - NDPI_LOG_DBG2(ndpi_struct, - "ndpi_parse_packet_line_info: HTTP response parsed: \"%.*s\"\n", - packet->http_response.len, packet->http_response.ptr); + if(packet->payload_packet_len >= 12) { + char buf[4]; + + /* Set server HTTP response code */ + strncpy(buf, (char*)&packet->payload[9], 3); + buf[3] = '\0'; + + flow->http.response_status_code = atoi(buf); + /* https://en.wikipedia.org/wiki/List_of_HTTP_status_codes */ + if((flow->http.response_status_code < 100) || (flow->http.response_status_code > 509)) + flow->http.response_status_code = 0; /* Out of range */ + } } /* "Server:" header line in HTTP response */ diff --git a/src/lib/protocols/http.c b/src/lib/protocols/http.c index 16b122d06..4ce80f9c9 100644 --- a/src/lib/protocols/http.c +++ b/src/lib/protocols/http.c @@ -26,6 +26,7 @@ #define NDPI_CURRENT_PROTO NDPI_PROTOCOL_HTTP #include "ndpi_api.h" +#include static void ndpi_int_http_add_connection(struct ndpi_detection_module_struct *ndpi_struct, struct ndpi_flow_struct *flow, @@ -54,7 +55,7 @@ static void ndpi_int_http_add_connection(struct ndpi_detection_module_struct *nd flow->packet.detected_protocol_stack[0] = NDPI_PROTOCOL_UNKNOWN, flow->packet.detected_protocol_stack[1] = NDPI_PROTOCOL_UNKNOWN; } } else { - if((!ndpi_struct->http_dont_dissect_response) && (flow->http.response_status_code[0] == '\0')) { + if((!ndpi_struct->http_dont_dissect_response) && (flow->http.response_status_code == 0)) { flow->http_upper_protocol = flow->detected_protocol_stack[0], flow->http_lower_protocol = flow->detected_protocol_stack[1]; flow->detected_protocol_stack[0] = NDPI_PROTOCOL_UNKNOWN, flow->detected_protocol_stack[1] = NDPI_PROTOCOL_UNKNOWN; flow->packet.detected_protocol_stack[0] = NDPI_PROTOCOL_UNKNOWN, flow->packet.detected_protocol_stack[1] = NDPI_PROTOCOL_UNKNOWN; @@ -211,10 +212,11 @@ static void check_content_type_and_change_protocol(struct ndpi_detection_module_ struct ndpi_packet_struct *packet = &flow->packet; u_int8_t a; - if((!ndpi_struct->http_dont_dissect_response) && flow->http_detected && (flow->http.response_status_code[0] != 0)) { + if((!ndpi_struct->http_dont_dissect_response) && flow->http_detected && (flow->http.response_status_code != 0)) { ndpi_set_detected_protocol(ndpi_struct, flow, flow->http_upper_protocol, flow->http_lower_protocol); #ifdef DEBUG - printf("[%s] [http_dont_dissect_response: %u]->> %s\n", __FUNCTION__, ndpi_struct->http_dont_dissect_response, flow->http.response_status_code); + printf("[%s] [http_dont_dissect_response: %u]->> %s\n", + __FUNCTION__, ndpi_struct->http_dont_dissect_response, flow->http.response_status_code); #endif return; } @@ -600,6 +602,20 @@ static void ndpi_check_http_tcp(struct ndpi_detection_module_struct *ndpi_struct if(packet->payload_packet_len >= 7 && memcmp(packet->payload, "HTTP/1.", 7) == 0) { NDPI_LOG_INFO(ndpi_struct, "found HTTP response\n"); + + if(packet->payload_packet_len >= 12) { + char buf[4]; + + /* Set server HTTP response code */ + strncpy(buf, (char*)&packet->payload[9], 3); + buf[3] = '\0'; + + flow->http.response_status_code = atoi(buf); + /* https://en.wikipedia.org/wiki/List_of_HTTP_status_codes */ + if((flow->http.response_status_code < 100) || (flow->http.response_status_code > 509)) + flow->http.response_status_code = 0; /* Out of range */ + } + ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_HTTP); check_content_type_and_change_protocol(ndpi_struct, flow); return; -- cgit v1.2.3 From 6a98aec2caa78c1c47157d288a008a84a9900959 Mon Sep 17 00:00:00 2001 From: Alfredo Cardigliano Date: Wed, 30 Jan 2019 08:44:32 +0000 Subject: Support for protocol file files of arbitrary length --- src/lib/ndpi_main.c | 53 ++++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 44 insertions(+), 9 deletions(-) (limited to 'src/lib') diff --git a/src/lib/ndpi_main.c b/src/lib/ndpi_main.c index 0b7af360d..1e0b0e201 100644 --- a/src/lib/ndpi_main.c +++ b/src/lib/ndpi_main.c @@ -2762,31 +2762,66 @@ int ndpi_handle_rule(struct ndpi_detection_module_struct *ndpi_mod, */ int ndpi_load_protocols_file(struct ndpi_detection_module_struct *ndpi_mod, char* path) { - FILE *fd = fopen(path, "r"); - int i; + FILE *fd; + char *buffer; + int chunk_len = 512, buffer_len = chunk_len, old_buffer_len; + int i, rc = -1; + + fd = fopen(path, "r"); if(fd == NULL) { NDPI_LOG_ERR(ndpi_mod, "Unable to open file %s [%s]", path, strerror(errno)); - return(-1); + goto error; + } + + buffer = ndpi_malloc(buffer_len); + + if(buffer == NULL) { + NDPI_LOG_ERR(ndpi_mod, "Memory allocation failure"); + goto close_fd; } while(fd) { - char buffer[512], *line; + char *line = buffer; + int line_len = buffer_len; + + while((line = fgets(line, line_len, fd)) != NULL && line[strlen(line)-1] != '\n') { + i = strlen(line); + old_buffer_len = buffer_len; + buffer_len += chunk_len; + + buffer = ndpi_realloc(buffer, old_buffer_len, buffer_len); - if(!(line = fgets(buffer, sizeof(buffer), fd))) + if(buffer == NULL) { + NDPI_LOG_ERR(ndpi_mod, "Memory allocation failure"); + goto close_fd; + } + + line = &buffer[i]; + line_len = chunk_len; + } + + if(!line) /* safety check */ break; - if(((i = strlen(line)) <= 1) || (line[0] == '#')) + i = strlen(buffer); + if((i <= 1) || (buffer[0] == '#')) continue; else - line[i-1] = '\0'; + buffer[i-1] = '\0'; - ndpi_handle_rule(ndpi_mod, line, 1); + ndpi_handle_rule(ndpi_mod, buffer, 1); } + rc = 0; + + free(buffer); + + close_fd: fclose(fd); - return(0); + error: + return(rc); } /* ******************************************************************** */ -- cgit v1.2.3 From 2ec41c17320622fe83dbba97ae8a2f4b40a30f34 Mon Sep 17 00:00:00 2001 From: cardigliano Date: Thu, 31 Jan 2019 09:12:04 +0100 Subject: Releasing old buffer if realloc fails --- src/lib/ndpi_main.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'src/lib') diff --git a/src/lib/ndpi_main.c b/src/lib/ndpi_main.c index 1e0b0e201..ea370ff7d 100644 --- a/src/lib/ndpi_main.c +++ b/src/lib/ndpi_main.c @@ -2763,7 +2763,7 @@ int ndpi_handle_rule(struct ndpi_detection_module_struct *ndpi_mod, */ int ndpi_load_protocols_file(struct ndpi_detection_module_struct *ndpi_mod, char* path) { FILE *fd; - char *buffer; + char *buffer, *old_buffer; int chunk_len = 512, buffer_len = chunk_len, old_buffer_len; int i, rc = -1; @@ -2787,13 +2787,15 @@ int ndpi_load_protocols_file(struct ndpi_detection_module_struct *ndpi_mod, char while((line = fgets(line, line_len, fd)) != NULL && line[strlen(line)-1] != '\n') { i = strlen(line); + old_buffer = buffer; old_buffer_len = buffer_len; buffer_len += chunk_len; - buffer = ndpi_realloc(buffer, old_buffer_len, buffer_len); + buffer = ndpi_realloc(old_buffer, old_buffer_len, buffer_len); if(buffer == NULL) { - NDPI_LOG_ERR(ndpi_mod, "Memory allocation failure"); + NDPI_LOG_ERR(ndpi_mod, "Memory allocation failure"); + free(old_buffer); goto close_fd; } -- cgit v1.2.3 From 476fbbfe4e141c199f0cd79258ab9085ee536e9e Mon Sep 17 00:00:00 2001 From: Vitaly Lavrov Date: Thu, 31 Jan 2019 11:27:40 +0300 Subject: Fix whatsapp signature detecting. --- src/lib/protocols/whatsapp.c | 34 +++++++++++++--------------------- 1 file changed, 13 insertions(+), 21 deletions(-) (limited to 'src/lib') diff --git a/src/lib/protocols/whatsapp.c b/src/lib/protocols/whatsapp.c index 6964a8e0e..608e6576e 100644 --- a/src/lib/protocols/whatsapp.c +++ b/src/lib/protocols/whatsapp.c @@ -26,34 +26,26 @@ void ndpi_search_whatsapp(struct ndpi_detection_module_struct *ndpi_struct, struct ndpi_flow_struct *flow) { struct ndpi_packet_struct *packet = &flow->packet; - u_int8_t whatsapp_sequence[] = { + static u_int8_t whatsapp_sequence[] = { 0x45, 0x44, 0x0, 0x01, 0x0, 0x0, 0x02, 0x08, 0x0, 0x57, 0x41, 0x02, 0x0, 0x0, 0x0 }; NDPI_LOG_DBG(ndpi_struct, "search WhatsApp\n"); - if(flow->l4.tcp.wa_matched_so_far == 0) { - if(memcmp(packet->payload, whatsapp_sequence, packet->payload_packet_len)) { - NDPI_EXCLUDE_PROTO(ndpi_struct, flow); - } else - flow->l4.tcp.wa_matched_so_far = packet->payload_packet_len; + if(flow->l4.tcp.wa_matched_so_far < sizeof(whatsapp_sequence)) { + size_t match_len = sizeof(whatsapp_sequence) - flow->l4.tcp.wa_matched_so_far; + if(packet->payload_packet_len < match_len) + match_len = packet->payload_packet_len; - return; - } else { - if(memcmp(packet->payload, &whatsapp_sequence[flow->l4.tcp.wa_matched_so_far], - sizeof(whatsapp_sequence)-flow->l4.tcp.wa_matched_so_far)) - NDPI_EXCLUDE_PROTO(ndpi_struct, flow); - else - ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_WHATSAPP, NDPI_PROTOCOL_UNKNOWN); - - return; - } - - if((packet->payload_packet_len > 240) - && (memcmp(packet->payload, whatsapp_sequence, sizeof(whatsapp_sequence)) == 0)) { - NDPI_LOG_INFO(ndpi_struct, "found WhatsApp\n"); - ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_WHATSAPP, NDPI_PROTOCOL_UNKNOWN); + if(!memcmp(packet->payload, &whatsapp_sequence[flow->l4.tcp.wa_matched_so_far], match_len)) { + flow->l4.tcp.wa_matched_so_far += match_len; + if(flow->l4.tcp.wa_matched_so_far == sizeof(whatsapp_sequence)) { + NDPI_LOG_INFO(ndpi_struct, "found WhatsApp\n"); + ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_WHATSAPP, NDPI_PROTOCOL_UNKNOWN); + } + return; + } } NDPI_EXCLUDE_PROTO(ndpi_struct, flow); -- cgit v1.2.3 From ddf0066c11c0df4e3bc9744df11f08dce676f36e Mon Sep 17 00:00:00 2001 From: Luca Deri Date: Thu, 7 Feb 2019 22:44:48 +0100 Subject: Aligned DNS to HTTP dissection --- example/ndpiReader.c | 2 +- src/include/ndpi_typedefs.h | 4 ++-- src/lib/ndpi_main.c | 4 ++-- src/lib/protocols/dns.c | 41 +++++++++++++++++++++-------------------- 4 files changed, 26 insertions(+), 25 deletions(-) (limited to 'src/lib') diff --git a/example/ndpiReader.c b/example/ndpiReader.c index 9cb4ad6f6..fe0ea879b 100644 --- a/example/ndpiReader.c +++ b/example/ndpiReader.c @@ -1459,7 +1459,7 @@ static void setupDetection(u_int16_t thread_id, pcap_t * pcap_handle) { ndpi_set_detection_preferences(ndpi_thread_info[thread_id].workflow->ndpi_struct, ndpi_pref_http_dont_dissect_response, 0); ndpi_set_detection_preferences(ndpi_thread_info[thread_id].workflow->ndpi_struct, - ndpi_pref_dns_dissect_response, 0); + ndpi_pref_dns_dont_dissect_response, 0); ndpi_set_detection_preferences(ndpi_thread_info[thread_id].workflow->ndpi_struct, ndpi_pref_enable_category_substring_match, 1); diff --git a/src/include/ndpi_typedefs.h b/src/include/ndpi_typedefs.h index 172763789..a5f1864e0 100644 --- a/src/include/ndpi_typedefs.h +++ b/src/include/ndpi_typedefs.h @@ -827,7 +827,7 @@ typedef enum { typedef enum { ndpi_pref_http_dont_dissect_response = 0, - ndpi_pref_dns_dissect_response, + ndpi_pref_dns_dont_dissect_response, ndpi_pref_direction_detect_disable, ndpi_pref_disable_metadata_export, ndpi_pref_enable_category_substring_match @@ -1008,7 +1008,7 @@ struct ndpi_detection_module_struct { ndpi_proto_defaults_t proto_defaults[NDPI_MAX_SUPPORTED_PROTOCOLS+NDPI_MAX_NUM_CUSTOM_PROTOCOLS]; - u_int8_t http_dont_dissect_response:1, dns_dissect_response:1, + u_int8_t http_dont_dissect_response:1, dns_dont_dissect_response:1, direction_detect_disable:1, /* disable internal detection of packet direction */ disable_metadata_export:1, /* No metadata is exported */ enable_category_substring_match:1 /* Default is perfect match */ diff --git a/src/lib/ndpi_main.c b/src/lib/ndpi_main.c index ea370ff7d..62914efbe 100644 --- a/src/lib/ndpi_main.c +++ b/src/lib/ndpi_main.c @@ -942,8 +942,8 @@ int ndpi_set_detection_preferences(struct ndpi_detection_module_struct *ndpi_mod ndpi_mod->http_dont_dissect_response = (u_int8_t)value; break; - case ndpi_pref_dns_dissect_response: - ndpi_mod->dns_dissect_response = (u_int8_t)value; + case ndpi_pref_dns_dont_dissect_response: + ndpi_mod->dns_dont_dissect_response = (u_int8_t)value; break; case ndpi_pref_direction_detect_disable: diff --git a/src/lib/protocols/dns.c b/src/lib/protocols/dns.c index 90be9544c..f77040020 100644 --- a/src/lib/protocols/dns.c +++ b/src/lib/protocols/dns.c @@ -36,9 +36,9 @@ static u_int16_t get16(int *i, const u_int8_t *payload) { u_int16_t v = *(u_int16_t*)&payload[*i]; - + (*i) += 2; - + return(ntohs(v)); } @@ -52,7 +52,7 @@ static u_int getNameLength(u_int i, const u_int8_t *payload, u_int payloadLen) { else { u_int8_t len = payload[i]; u_int8_t off = len + 1; - + if(off == 0) /* Bad packet */ return(0); else @@ -66,7 +66,7 @@ void ndpi_search_dns(struct ndpi_detection_module_struct *ndpi_struct, struct nd int x; u_int8_t is_query; u_int16_t s_port = 0, d_port = 0; - + NDPI_LOG_DBG(ndpi_struct, "search DNS\n"); if(flow->packet.udp != NULL) { @@ -118,7 +118,7 @@ void ndpi_search_dns(struct ndpi_detection_module_struct *ndpi_struct, struct nd if(flow->packet.payload[x] == '\0') { x++; flow->protos.dns.query_type = get16(&x, flow->packet.payload); -#ifdef DNS_DEBUG +#ifdef DNS_DEBUG NDPI_LOG_DBG2(ndpi_struct, "query_type=%2d\n", flow->protos.dns.query_type); #endif break; @@ -128,7 +128,6 @@ void ndpi_search_dns(struct ndpi_detection_module_struct *ndpi_struct, struct nd } } else invalid = 1; - } else { /* DNS Reply */ @@ -140,15 +139,15 @@ void ndpi_search_dns(struct ndpi_detection_module_struct *ndpi_struct, struct nd || ((dns_header.additional_rrs > 0) && (dns_header.additional_rrs <= NDPI_MAX_DNS_REQUESTS))) ) { /* This is a good reply */ - if(ndpi_struct->dns_dissect_response) { + if(ndpi_struct->dns_dont_dissect_response == 0) { x++; - + if(flow->packet.payload[x] != '\0') { while((x < flow->packet.payload_packet_len) && (flow->packet.payload[x] != '\0')) { x++; } - + x++; } @@ -160,7 +159,7 @@ void ndpi_search_dns(struct ndpi_detection_module_struct *ndpi_struct, struct nd for(num = 0; num < dns_header.num_answers; num++) { u_int16_t data_len; - + if((x+6) >= flow->packet.payload_packet_len) { break; } @@ -169,7 +168,7 @@ void ndpi_search_dns(struct ndpi_detection_module_struct *ndpi_struct, struct nd break; } else x += data_len; - + rsp_type = get16(&x, flow->packet.payload); flow->protos.dns.rsp_type = rsp_type; break; @@ -199,9 +198,11 @@ void ndpi_search_dns(struct ndpi_detection_module_struct *ndpi_struct, struct nd off++; } - if(is_query && ndpi_struct->dns_dissect_response) - return; /* The response will set the verdict */ - + if(is_query && (ndpi_struct->dns_dont_dissect_response == 0)) { + // dpi_set_detected_protocol(ndpi_struct, flow, (d_port == 5355) ? NDPI_PROTOCOL_LLMNR : NDPI_PROTOCOL_DNS, NDPI_PROTOCOL_UNKNOWN); + return; /* The response will set the verdict */ + } + flow->host_server_name[j] = '\0'; flow->protos.dns.num_queries = (u_int8_t)dns_header.num_queries, @@ -209,33 +210,33 @@ void ndpi_search_dns(struct ndpi_detection_module_struct *ndpi_struct, struct nd if(j > 0) { ndpi_protocol_match_result ret_match; - - ndpi_match_host_subprotocol(ndpi_struct, flow, + + ndpi_match_host_subprotocol(ndpi_struct, flow, (char *)flow->host_server_name, strlen((const char*)flow->host_server_name), &ret_match, NDPI_PROTOCOL_DNS); } - + #ifdef DNS_DEBUG NDPI_LOG_DBG2(ndpi_struct, "[num_queries=%d][num_answers=%d][reply_code=%u][rsp_type=%u][host_server_name=%s]\n", flow->protos.dns.num_queries, flow->protos.dns.num_answers, flow->protos.dns.reply_code, flow->protos.dns.rsp_type, flow->host_server_name ); #endif - + if(flow->packet.detected_protocol_stack[0] == NDPI_PROTOCOL_UNKNOWN) { /** Do not set the protocol with DNS if ndpi_match_host_subprotocol() has matched a subprotocol **/ - NDPI_LOG_INFO(ndpi_struct, "found DNS\n"); + NDPI_LOG_INFO(ndpi_struct, "found DNS\n"); ndpi_set_detected_protocol(ndpi_struct, flow, (d_port == 5355) ? NDPI_PROTOCOL_LLMNR : NDPI_PROTOCOL_DNS, NDPI_PROTOCOL_UNKNOWN); } else { NDPI_EXCLUDE_PROTO(ndpi_struct, flow); } } - } + } } void init_dns_dissector(struct ndpi_detection_module_struct *ndpi_struct, u_int32_t *id, NDPI_PROTOCOL_BITMASK *detection_bitmask) -- cgit v1.2.3 From 0c4fada265b7932688c0bec6fa8229b9907f5831 Mon Sep 17 00:00:00 2001 From: Luca Deri Date: Sun, 10 Feb 2019 23:52:24 +0100 Subject: Fix for avoid wrong substring match to happen --- src/lib/ndpi_main.c | 43 +++++++++++++++++++++++++++++++++---------- 1 file changed, 33 insertions(+), 10 deletions(-) (limited to 'src/lib') diff --git a/src/lib/ndpi_main.c b/src/lib/ndpi_main.c index 62914efbe..8c9159ef2 100644 --- a/src/lib/ndpi_main.c +++ b/src/lib/ndpi_main.c @@ -1991,21 +1991,38 @@ static int ac_match_handler(AC_MATCH_t *m, AC_TEXT_t *txt, AC_REP_t *match) { m->match_num, m->patterns->astring); #endif + { + char *whatfound = strstr(buf, m->patterns->astring); + +#ifdef MATCH_DEBUG + printf("[NDPI] %s() [searching=%s][pattern=%s][%s][%c]\n", + __FUNCTION__, buf, m->patterns->astring, + whatfound ? whatfound : "", + whatfound[-1]); +#endif + + /* + The patch below allows in case of pattern ws.amazon.com + to avoid matching aws.amazon.com whereas a.ws.amazon.com + has to match + */ + if(whatfound && (whatfound != buf) && (whatfound[-1] != '.')) + return(0); + } + /* Return 1 for stopping to the first match. We might consider searching for the more specific match, paying more cpu cycles. */ - memcpy(match, &m->patterns[0].rep, sizeof(AC_REP_t)); if(((buf_len >= min_len) && (strncmp(&buf[buf_len-min_len], m->patterns->astring, min_len) == 0)) || (strncmp(buf, m->patterns->astring, min_len) == 0) /* begins with */ - ) - { + ) { #ifdef MATCH_DEBUG printf("Found match [%s][%s] [len: %u][proto_id: %u]\n", - buf, m->patterns->astring, min_len, *matching_protocol_id); + buf, m->patterns->astring, min_len , *matching_protocol_id); #endif return(1); /* If the pattern found matches the string at the beginning we stop here */ } else @@ -2301,8 +2318,8 @@ int ndpi_match_string(void *_automa, char *string_to_match) { int ndpi_match_string_id(void *_automa, char *string_to_match, unsigned long *id) { AC_TEXT_t ac_input_text; AC_AUTOMATA_t *automa = (AC_AUTOMATA_t*)_automa; - AC_REP_t match = { NDPI_PROTOCOL_UNKNOWN, NDPI_PROTOCOL_CATEGORY_UNSPECIFIED, NDPI_PROTOCOL_UNRATED }; - + AC_REP_t match = { NDPI_PROTOCOL_UNKNOWN, NDPI_PROTOCOL_CATEGORY_UNSPECIFIED, NDPI_PROTOCOL_UNRATED }; + *id = -1; if((automa == NULL) || (string_to_match == NULL) @@ -2340,9 +2357,12 @@ static int hyperscanCustomEventHandler(unsigned int id, static int ndpi_match_custom_category(struct ndpi_detection_module_struct *ndpi_struct, char *name, unsigned long *id) { - /* printf("[NDPI] %s(%s)\n", __FUNCTION__, name); */ +#ifdef DEBUG + printf("[NDPI] %s(%s) [enable_category_substring_match: %u]\n", + __FUNCTION__, name, ndpi_struct->enable_category_substring_match); +#endif - if(!ndpi_struct->enable_category_substring_match) { + if(ndpi_struct->enable_category_substring_match == 0) { if(ndpi_struct->custom_categories.hostnames_hash == NULL) return(-1); else { @@ -2731,7 +2751,8 @@ int ndpi_handle_rule(struct ndpi_detection_module_struct *ndpi_mod, if(sscanf(value, "%u-%u", (u_int32_t *)&range.port_low, (u_int32_t *)&range.port_high) != 2) range.port_low = range.port_high = atoi(&elem[4]); if(do_add) - addDefaultPort(ndpi_mod, &range, def, 1 /* Custom user proto */, is_tcp ? &ndpi_mod->tcpRoot : &ndpi_mod->udpRoot, __FUNCTION__,__LINE__); + addDefaultPort(ndpi_mod, &range, def, 1 /* Custom user proto */, + is_tcp ? &ndpi_mod->tcpRoot : &ndpi_mod->udpRoot, __FUNCTION__,__LINE__); else removeDefaultPort(&range, def, is_tcp ? &ndpi_mod->tcpRoot : &ndpi_mod->udpRoot); } else if(is_ip) { @@ -4385,6 +4406,8 @@ int ndpi_fill_ip_protocol_category(struct ndpi_detection_module_struct *ndpi_str return 0; } +/* ********************************************************************************* */ + void ndpi_fill_protocol_category(struct ndpi_detection_module_struct *ndpi_struct, struct ndpi_flow_struct *flow, ndpi_protocol *ret) { @@ -4435,7 +4458,7 @@ ndpi_protocol ndpi_detection_process_packet(struct ndpi_detection_module_struct if(ndpi_struct->ndpi_log_level >= NDPI_LOG_TRACE) NDPI_LOG(flow ? flow->detected_protocol_stack[0]:NDPI_PROTOCOL_UNKNOWN, - ndpi_struct, NDPI_LOG_TRACE, "START packet processing\n"); + ndpi_struct, NDPI_LOG_TRACE, "START packet processing\n"); if(flow == NULL) return(ret); -- cgit v1.2.3 From a035763afd3a2eb3d959d67152ea309cdb16d448 Mon Sep 17 00:00:00 2001 From: emanuele-f Date: Tue, 12 Feb 2019 17:28:30 +0100 Subject: Change ndpi_fill_ip_protocol_category to pass IPs explicitly --- src/include/ndpi_api.h | 5 ++++- src/lib/ndpi_main.c | 13 +++++++------ 2 files changed, 11 insertions(+), 7 deletions(-) (limited to 'src/lib') diff --git a/src/include/ndpi_api.h b/src/include/ndpi_api.h index e09c91c8e..386d306fd 100644 --- a/src/include/ndpi_api.h +++ b/src/include/ndpi_api.h @@ -740,8 +740,11 @@ extern "C" { char *name, ndpi_protocol_category_t category); int ndpi_enable_loaded_categories(struct ndpi_detection_module_struct *ndpi_struct); int ndpi_fill_ip_protocol_category(struct ndpi_detection_module_struct *ndpi_struct, - const struct ndpi_iphdr *iph, + u_int32_t saddr, + u_int32_t daddr, ndpi_protocol *ret); + int ndpi_match_custom_category(struct ndpi_detection_module_struct *ndpi_struct, + char *name, unsigned long *id); void ndpi_fill_protocol_category(struct ndpi_detection_module_struct *ndpi_struct, struct ndpi_flow_struct *flow, ndpi_protocol *ret); diff --git a/src/lib/ndpi_main.c b/src/lib/ndpi_main.c index 8c9159ef2..fe63c8093 100644 --- a/src/lib/ndpi_main.c +++ b/src/lib/ndpi_main.c @@ -2355,7 +2355,7 @@ static int hyperscanCustomEventHandler(unsigned int id, /* *********************************************** */ -static int ndpi_match_custom_category(struct ndpi_detection_module_struct *ndpi_struct, +int ndpi_match_custom_category(struct ndpi_detection_module_struct *ndpi_struct, char *name, unsigned long *id) { #ifdef DEBUG printf("[NDPI] %s(%s) [enable_category_substring_match: %u]\n", @@ -4379,19 +4379,20 @@ int ndpi_enable_loaded_categories(struct ndpi_detection_module_struct *ndpi_str) /* ********************************************************************************* */ int ndpi_fill_ip_protocol_category(struct ndpi_detection_module_struct *ndpi_struct, - const struct ndpi_iphdr *iph, + u_int32_t saddr, + u_int32_t daddr, ndpi_protocol *ret) { if(ndpi_struct->custom_categories.categories_loaded) { prefix_t prefix; patricia_node_t *node; - + /* Make sure all in network byte order otherwise compares wont work */ - fill_prefix_v4(&prefix, (struct in_addr *)&iph->saddr, + fill_prefix_v4(&prefix, (struct in_addr *)&saddr, 32, ((patricia_tree_t*)ndpi_struct->protocols_ptree)->maxbits); node = ndpi_patricia_search_best(ndpi_struct->custom_categories.ipAddresses, &prefix); if(!node) { - fill_prefix_v4(&prefix, (struct in_addr *)&iph->daddr, + fill_prefix_v4(&prefix, (struct in_addr *)&daddr, 32, ((patricia_tree_t*)ndpi_struct->protocols_ptree)->maxbits); node = ndpi_patricia_search_best(ndpi_struct->custom_categories.ipAddresses, &prefix); } @@ -4413,7 +4414,7 @@ void ndpi_fill_protocol_category(struct ndpi_detection_module_struct *ndpi_struc ndpi_protocol *ret) { if(ndpi_struct->custom_categories.categories_loaded) { if(flow->packet.iph) { - if(ndpi_fill_ip_protocol_category(ndpi_struct, flow->packet.iph, ret)) { + if(ndpi_fill_ip_protocol_category(ndpi_struct, flow->packet.iph->saddr, flow->packet.iph->daddr, ret)) { flow->category = ret->category; return; } -- cgit v1.2.3 From dc8d582b1813db1539c1ebc05f5614b634806192 Mon Sep 17 00:00:00 2001 From: Luca Deri Date: Thu, 14 Feb 2019 10:31:42 +0100 Subject: Fixes #671 --- src/lib/ndpi_main.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'src/lib') diff --git a/src/lib/ndpi_main.c b/src/lib/ndpi_main.c index fe63c8093..f96f0e763 100644 --- a/src/lib/ndpi_main.c +++ b/src/lib/ndpi_main.c @@ -2006,7 +2006,9 @@ static int ac_match_handler(AC_MATCH_t *m, AC_TEXT_t *txt, AC_REP_t *match) { to avoid matching aws.amazon.com whereas a.ws.amazon.com has to match */ - if(whatfound && (whatfound != buf) && (whatfound[-1] != '.')) + if(whatfound && (whatfound != buf) + && strchr(m->patterns->astring, '.') /* The matched pattern has a . (e.g. numeric or sym IPs) */ + && (whatfound[-1] != '.')) return(0); } -- cgit v1.2.3 From ab3833a708fbd56e7edc72169563d26dffeb14de Mon Sep 17 00:00:00 2001 From: emanuele-f Date: Tue, 19 Feb 2019 17:17:02 +0100 Subject: Fix invalid TCP DNS dissection --- src/lib/protocols/dns.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) (limited to 'src/lib') diff --git a/src/lib/protocols/dns.c b/src/lib/protocols/dns.c index f77040020..b636f3e89 100644 --- a/src/lib/protocols/dns.c +++ b/src/lib/protocols/dns.c @@ -63,7 +63,7 @@ static u_int getNameLength(u_int i, const u_int8_t *payload, u_int payloadLen) { /* *********************************************** */ void ndpi_search_dns(struct ndpi_detection_module_struct *ndpi_struct, struct ndpi_flow_struct *flow) { - int x; + int x, payload_offset; u_int8_t is_query; u_int16_t s_port = 0, d_port = 0; @@ -72,16 +72,18 @@ void ndpi_search_dns(struct ndpi_detection_module_struct *ndpi_struct, struct nd if(flow->packet.udp != NULL) { s_port = ntohs(flow->packet.udp->source); d_port = ntohs(flow->packet.udp->dest); - x = 0; + payload_offset = 0; } else if(flow->packet.tcp != NULL) /* pkt size > 512 bytes */ { s_port = ntohs(flow->packet.tcp->source); d_port = ntohs(flow->packet.tcp->dest); - x = 2; + payload_offset = 2; } else { NDPI_EXCLUDE_PROTO(ndpi_struct, flow); return; } + x = payload_offset; + if((s_port == 53 || d_port == 53 || d_port == 5355) && (flow->packet.payload_packet_len > sizeof(struct ndpi_dns_packet_header)+x)) { struct ndpi_dns_packet_header dns_header; @@ -184,8 +186,7 @@ void ndpi_search_dns(struct ndpi_detection_module_struct *ndpi_struct, struct nd } /* extract host name server */ - int j = 0, max_len = sizeof(flow->host_server_name)-1, off = sizeof(struct ndpi_dns_packet_header) + 1; - + int j = 0, max_len = sizeof(flow->host_server_name)-1, off = sizeof(struct ndpi_dns_packet_header) + 1 + payload_offset; while(off < flow->packet.payload_packet_len && flow->packet.payload[off] != '\0') { flow->host_server_name[j] = flow->packet.payload[off]; if(j < max_len) { -- cgit v1.2.3 From 47669abd441888003c5ae815775c84763fd6f1c2 Mon Sep 17 00:00:00 2001 From: Luca Deri Date: Thu, 21 Feb 2019 23:29:06 +0100 Subject: Added extra check to avoid nDPI wrong matches --- src/lib/ndpi_main.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'src/lib') diff --git a/src/lib/ndpi_main.c b/src/lib/ndpi_main.c index f96f0e763..b0c25e8c5 100644 --- a/src/lib/ndpi_main.c +++ b/src/lib/ndpi_main.c @@ -2007,8 +2007,10 @@ static int ac_match_handler(AC_MATCH_t *m, AC_TEXT_t *txt, AC_REP_t *match) { has to match */ if(whatfound && (whatfound != buf) + && (m->patterns->astring[0] != '.') /* The searched patter does not start with . */ && strchr(m->patterns->astring, '.') /* The matched pattern has a . (e.g. numeric or sym IPs) */ - && (whatfound[-1] != '.')) + && (whatfound[-1] != '.') + ) return(0); } -- cgit v1.2.3 From de16b01bb2c56f0b8b963e6a8ab404555314c458 Mon Sep 17 00:00:00 2001 From: Luca Deri Date: Sun, 24 Feb 2019 11:03:06 +0100 Subject: Improved MDNS dissection --- src/lib/protocols/mdns_proto.c | 29 ++++++++++++----------------- 1 file changed, 12 insertions(+), 17 deletions(-) (limited to 'src/lib') diff --git a/src/lib/protocols/mdns_proto.c b/src/lib/protocols/mdns_proto.c index 00c7c8748..75eab720b 100644 --- a/src/lib/protocols/mdns_proto.c +++ b/src/lib/protocols/mdns_proto.c @@ -63,17 +63,15 @@ static int ndpi_int_check_mdns_payload(struct ndpi_detection_module_struct struct ndpi_packet_struct *packet = &flow->packet; struct mdns_header *h = (struct mdns_header*)packet->payload; u_int16_t questions = ntohs(h->questions), answers = ntohs(h->answers); + + if((questions > NDPI_MAX_MDNS_REQUESTS) + || (answers > NDPI_MAX_MDNS_REQUESTS)) + return(0); - if(((packet->payload[2] & 0x80) == 0) - && (questions <= NDPI_MAX_MDNS_REQUESTS) - && (answers <= NDPI_MAX_MDNS_REQUESTS)) { + if((packet->payload[2] & 0x80) == 0) { NDPI_LOG_INFO(ndpi_struct, "found MDNS with question query\n"); return 1; - } - else if(((packet->payload[2] & 0x80) != 0) - && (questions == 0) - && (answers <= NDPI_MAX_MDNS_REQUESTS) - && (answers != 0)) { + } else if((packet->payload[2] & 0x80) != 0) { char answer[256]; int i, j, len; @@ -100,8 +98,6 @@ static int ndpi_int_check_mdns_payload(struct ndpi_detection_module_struct void ndpi_search_mdns(struct ndpi_detection_module_struct *ndpi_struct, struct ndpi_flow_struct *flow) { struct ndpi_packet_struct *packet = &flow->packet; - u_int16_t dport; - NDPI_LOG_DBG(ndpi_struct, "search MDNS\n"); /** @@ -111,15 +107,13 @@ void ndpi_search_mdns(struct ndpi_detection_module_struct *ndpi_struct, struct n /* check if UDP packet */ if(packet->udp != NULL) { /* read destination port */ - dport = ntohs(packet->udp->dest); + u_int16_t sport = ntohs(packet->udp->source); + u_int16_t dport = ntohs(packet->udp->dest); /* check standard MDNS ON port 5353 */ - if(dport == 5353 && packet->payload_packet_len >= 12) { - /* mdns protocol must have destination address 224.0.0.251 */ - if(packet->iph != NULL /* && ntohl(packet->iph->daddr) == 0xe00000fb */) { - - NDPI_LOG_INFO(ndpi_struct, "found MDNS with destination address 224.0.0.251 (=0xe00000fb)\n"); - + if(((dport == 5353) || (sport == 5353)) + && (packet->payload_packet_len >= 12)) { + if(packet->iph != NULL) { if(ndpi_int_check_mdns_payload(ndpi_struct, flow) == 1) { ndpi_int_mdns_add_connection(ndpi_struct, flow); return; @@ -141,6 +135,7 @@ void ndpi_search_mdns(struct ndpi_detection_module_struct *ndpi_struct, struct n #endif } } + NDPI_EXCLUDE_PROTO(ndpi_struct, flow); } -- cgit v1.2.3 From a99754d88bd1386536162d434304b61048da3f22 Mon Sep 17 00:00:00 2001 From: Vitaly Lavrov Date: Wed, 27 Feb 2019 12:21:33 +0300 Subject: More accurate hostname verification in a DNS query. --- src/lib/protocols/dns.c | 44 ++++++++++++++++++++++++++++++-------------- 1 file changed, 30 insertions(+), 14 deletions(-) (limited to 'src/lib') diff --git a/src/lib/protocols/dns.c b/src/lib/protocols/dns.c index b636f3e89..57213b889 100644 --- a/src/lib/protocols/dns.c +++ b/src/lib/protocols/dns.c @@ -59,7 +59,19 @@ static u_int getNameLength(u_int i, const u_int8_t *payload, u_int payloadLen) { return(off + getNameLength(i+off, payload, payloadLen)); } } +/* + allowed chars for dns names A-Z 0-9 _ - + Perl script for generation map: + my @M; + for(my $ch=0; $ch < 256; $ch++) { + $M[$ch >> 5] |= 1 << ($ch & 0x1f) if chr($ch) =~ /[a-z0-9_-]/i; + } + print join(',', map { sprintf "0x%08x",$_ } @M),"\n"; + */ +static uint32_t dns_validchar[8] = { + 0x00000000,0x03ff2000,0x87fffffe,0x07fffffe,0,0,0,0 +}; /* *********************************************** */ void ndpi_search_dns(struct ndpi_detection_module_struct *ndpi_struct, struct ndpi_flow_struct *flow) { @@ -108,6 +120,7 @@ void ndpi_search_dns(struct ndpi_detection_module_struct *ndpi_struct, struct nd invalid = 1; if(!invalid) { + int j = 0, max_len, off; if(is_query) { /* DNS Request */ if((dns_header.num_queries > 0) && (dns_header.num_queries <= NDPI_MAX_DNS_REQUESTS) @@ -186,28 +199,31 @@ void ndpi_search_dns(struct ndpi_detection_module_struct *ndpi_struct, struct nd } /* extract host name server */ - int j = 0, max_len = sizeof(flow->host_server_name)-1, off = sizeof(struct ndpi_dns_packet_header) + 1 + payload_offset; - while(off < flow->packet.payload_packet_len && flow->packet.payload[off] != '\0') { - flow->host_server_name[j] = flow->packet.payload[off]; - if(j < max_len) { - if(flow->host_server_name[j] < ' ') - flow->host_server_name[j] = '.'; - j++; - } else - break; - - off++; + max_len = sizeof(flow->host_server_name)-1; + off = sizeof(struct ndpi_dns_packet_header) + payload_offset; + + while(j < max_len && off < flow->packet.payload_packet_len && flow->packet.payload[off] != '\0') { + uint8_t c,cl = flow->packet.payload[off++]; + if( (cl & 0xc0) != 0 || // we not support compressed names in query + off + cl >= flow->packet.payload_packet_len) { + j = 0; break; + } + if(j && j < max_len) flow->host_server_name[j++] = '.'; + while(j < max_len && cl != 0) { + c = flow->packet.payload[off++]; + flow->host_server_name[j++] = dns_validchar[c >> 5] & (1 << (c & 0x1f)) ? c:'_'; + cl--; + } } + flow->host_server_name[j] = '\0'; if(is_query && (ndpi_struct->dns_dont_dissect_response == 0)) { // dpi_set_detected_protocol(ndpi_struct, flow, (d_port == 5355) ? NDPI_PROTOCOL_LLMNR : NDPI_PROTOCOL_DNS, NDPI_PROTOCOL_UNKNOWN); return; /* The response will set the verdict */ } - - flow->host_server_name[j] = '\0'; flow->protos.dns.num_queries = (u_int8_t)dns_header.num_queries, - flow->protos.dns.num_answers = (u_int8_t) (dns_header.num_answers + dns_header.authority_rrs + dns_header.additional_rrs); + flow->protos.dns.num_answers = (u_int8_t) (dns_header.num_answers + dns_header.authority_rrs + dns_header.additional_rrs); if(j > 0) { ndpi_protocol_match_result ret_match; -- cgit v1.2.3 From d283538146c64912a50b177b073b9b5c85c270c9 Mon Sep 17 00:00:00 2001 From: Luca Date: Fri, 1 Mar 2019 12:56:33 +0100 Subject: Fixed false positive mining detection --- src/lib/protocols/mining.c | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) (limited to 'src/lib') diff --git a/src/lib/protocols/mining.c b/src/lib/protocols/mining.c index ec094e7d3..b4361e270 100644 --- a/src/lib/protocols/mining.c +++ b/src/lib/protocols/mining.c @@ -44,9 +44,12 @@ void ndpi_search_mining_tcp(struct ndpi_detection_module_struct *ndpi_struct, if((*to_match == magic) || (*to_match == magic1)) { ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_MINING, NDPI_PROTOCOL_UNKNOWN); } - } if(ndpi_strnstr((const char *)packet->payload, "\"eth1.0\"", packet->payload_packet_len) - || ndpi_strnstr((const char *)packet->payload, "\"worker\"", packet->payload_packet_len) - || ndpi_strnstr((const char *)packet->payload, "\"id\"", packet->payload_packet_len)) { + } if(ndpi_strnstr((const char *)packet->payload, "{", packet->payload_packet_len) + && ( + ndpi_strnstr((const char *)packet->payload, "\"eth1.0\"", packet->payload_packet_len) + || ndpi_strnstr((const char *)packet->payload, "\"worker\":", packet->payload_packet_len) + /* || ndpi_strnstr((const char *)packet->payload, "\"id\":", packet->payload_packet_len) - Removed as too generic */ + )) { /* Ethereum @@ -55,9 +58,12 @@ void ndpi_search_mining_tcp(struct ndpi_detection_module_struct *ndpi_struct, {"worker": "", "jsonrpc": "2.0", "params": [], "id": 3, "method": "eth_getWork"} */ ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_MINING, NDPI_PROTOCOL_UNKNOWN); - } else if(ndpi_strnstr((const char *)packet->payload, "\"method\"", packet->payload_packet_len) - || ndpi_strnstr((const char *)packet->payload, "\"blob\"", packet->payload_packet_len) - || ndpi_strnstr((const char *)packet->payload, "\"id\"", packet->payload_packet_len)) { + } else if(ndpi_strnstr((const char *)packet->payload, "{", packet->payload_packet_len) + && (ndpi_strnstr((const char *)packet->payload, "\"method\":", packet->payload_packet_len) + || ndpi_strnstr((const char *)packet->payload, "\"blob\":", packet->payload_packet_len) + /* || ndpi_strnstr((const char *)packet->payload, "\"id\":", packet->payload_packet_len) - Removed as too generic */ + ) + ) { /* ZCash -- cgit v1.2.3 From c339211ac7043c9fe6f17971b4f7ef8c3b5ec9ab Mon Sep 17 00:00:00 2001 From: Luca Deri Date: Sun, 3 Mar 2019 21:07:54 +0100 Subject: Re-Added TikTok/Musical.ly protocol --- src/include/ndpi_protocol_ids.h | 2 +- src/lib/ndpi_content_match.c.inc | 2 ++ src/lib/ndpi_main.c | 9 ++------- 3 files changed, 5 insertions(+), 8 deletions(-) (limited to 'src/lib') diff --git a/src/include/ndpi_protocol_ids.h b/src/include/ndpi_protocol_ids.h index 7545de436..1d9c1d428 100644 --- a/src/include/ndpi_protocol_ids.h +++ b/src/include/ndpi_protocol_ids.h @@ -85,7 +85,7 @@ typedef enum { NDPI_PROTOCOL_XBOX = 47, NDPI_PROTOCOL_QQ = 48, - NDPI_PROTOCOL_FREE_49 = 49, /* Free */ + NDPI_PROTOCOL_TIKTOK = 49, NDPI_PROTOCOL_RTSP = 50, NDPI_PROTOCOL_MAIL_IMAPS = 51, NDPI_PROTOCOL_ICECAST = 52, diff --git a/src/lib/ndpi_content_match.c.inc b/src/lib/ndpi_content_match.c.inc index c0cba0b89..b72c697e6 100644 --- a/src/lib/ndpi_content_match.c.inc +++ b/src/lib/ndpi_content_match.c.inc @@ -8422,6 +8422,8 @@ ndpi_protocol_match host_match[] = { { "signal.org", NULL, "signal\\.org" TLD, "Signal", NDPI_PROTOCOL_SIGNAL, NDPI_PROTOCOL_CATEGORY_CHAT, NDPI_PROTOCOL_ACCEPTABLE }, { "whispersystems.org", NULL, "whispersystems\\.org" TLD, "Signal", NDPI_PROTOCOL_SIGNAL, NDPI_PROTOCOL_CATEGORY_CHAT, NDPI_PROTOCOL_ACCEPTABLE }, + { "musical.ly", NULL, "musical\\.ly" TLD, "TikTok", NDPI_PROTOCOL_TIKTOK, NDPI_PROTOCOL_CATEGORY_SOCIAL_NETWORK, NDPI_PROTOCOL_ACCEPTABLE }, + { "muscdn.com", NULL, "muscndl\\.com" TLD, "TikTok", NDPI_PROTOCOL_TIKTOK, NDPI_PROTOCOL_CATEGORY_SOCIAL_NETWORK, NDPI_PROTOCOL_ACCEPTABLE }, { NULL, NULL, NULL, 0 } }; diff --git a/src/lib/ndpi_main.c b/src/lib/ndpi_main.c index b0c25e8c5..f56526e61 100644 --- a/src/lib/ndpi_main.c +++ b/src/lib/ndpi_main.c @@ -1198,14 +1198,9 @@ static void ndpi_init_protocol_defaults(struct ndpi_detection_module_struct *ndp no_master, "SkypeCall", NDPI_PROTOCOL_CATEGORY_VOIP, ndpi_build_default_ports(ports_a, 0, 0, 0, 0, 0) /* TCP */, ndpi_build_default_ports(ports_b, 0, 0, 0, 0, 0) /* UDP */); - ndpi_set_proto_defaults(ndpi_mod, NDPI_PROTOCOL_ACCEPTABLE, NDPI_PROTOCOL_FREE_49, + ndpi_set_proto_defaults(ndpi_mod, NDPI_PROTOCOL_ACCEPTABLE, NDPI_PROTOCOL_TIKTOK, 0 /* can_have_a_subprotocol */, no_master, - no_master, "Free_49", NDPI_PROTOCOL_CATEGORY_VOIP, - ndpi_build_default_ports(ports_a, 0, 0, 0, 0, 0) /* TCP */, - ndpi_build_default_ports(ports_b, 0, 0, 0, 0, 0) /* UDP */); - ndpi_set_proto_defaults(ndpi_mod, NDPI_PROTOCOL_ACCEPTABLE, NDPI_PROTOCOL_FREE_49, - 0 /* can_have_a_subprotocol */, no_master, - no_master, "SkypeCall", NDPI_PROTOCOL_CATEGORY_VOIP, + no_master, "TikTok", NDPI_PROTOCOL_CATEGORY_SOCIAL_NETWORK, ndpi_build_default_ports(ports_a, 0, 0, 0, 0, 0) /* TCP */, ndpi_build_default_ports(ports_b, 0, 0, 0, 0, 0) /* UDP */); ndpi_set_proto_defaults(ndpi_mod, NDPI_PROTOCOL_ACCEPTABLE, NDPI_PROTOCOL_TEREDO, -- cgit v1.2.3 From 59f683858b2b8f28e88b9348b75fef5164665964 Mon Sep 17 00:00:00 2001 From: Simone Mainardi Date: Mon, 4 Mar 2019 18:48:38 +0100 Subject: Fixes possibly broken dissection of invalid DNS responses --- src/lib/protocols/dns.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'src/lib') diff --git a/src/lib/protocols/dns.c b/src/lib/protocols/dns.c index b636f3e89..e282eb4d3 100644 --- a/src/lib/protocols/dns.c +++ b/src/lib/protocols/dns.c @@ -177,7 +177,8 @@ void ndpi_search_dns(struct ndpi_detection_module_struct *ndpi_struct, struct nd } } } - } + } else + invalid = 1; } if(invalid) { -- cgit v1.2.3 From 03b0aa7185261f34dc60f1ac72ff142c5758f20d Mon Sep 17 00:00:00 2001 From: Luca Deri Date: Fri, 8 Mar 2019 15:58:21 +0000 Subject: Added new categories - NDPI_PROTOCOL_CATEGORY_MUSIC - NDPI_PROTOCOL_CATEGORY_VIDEO - NDPI_PROTOCOL_CATEGORY_SHOPPING - NDPI_PROTOCOL_CATEGORY_PRODUCTIVITY - NDPI_PROTOCOL_CATEGORY_FILE_SHARING Protocol vs category rework --- src/include/ndpi_typedefs.h | 7 ++++ src/lib/ndpi_content_match.c.inc | 84 ++++++++++++++++++++-------------------- src/lib/ndpi_main.c | 34 ++++++++-------- 3 files changed, 65 insertions(+), 60 deletions(-) (limited to 'src/lib') diff --git a/src/include/ndpi_typedefs.h b/src/include/ndpi_typedefs.h index a5f1864e0..917aa6827 100644 --- a/src/include/ndpi_typedefs.h +++ b/src/include/ndpi_typedefs.h @@ -801,6 +801,13 @@ typedef enum { NDPI_CONTENT_CATEGORY_WINDOWSMEDIA, NDPI_CONTENT_CATEGORY_WEBM, + /* Further categories... */ + NDPI_PROTOCOL_CATEGORY_MUSIC, + NDPI_PROTOCOL_CATEGORY_VIDEO, + NDPI_PROTOCOL_CATEGORY_SHOPPING, + NDPI_PROTOCOL_CATEGORY_PRODUCTIVITY, + NDPI_PROTOCOL_CATEGORY_FILE_SHARING, + /* Some custom categories */ CUSTOM_CATEGORY_MINING = 99, CUSTOM_CATEGORY_MALWARE = 100, diff --git a/src/lib/ndpi_content_match.c.inc b/src/lib/ndpi_content_match.c.inc index c0cba0b89..8daa8a825 100644 --- a/src/lib/ndpi_content_match.c.inc +++ b/src/lib/ndpi_content_match.c.inc @@ -8031,11 +8031,11 @@ https://www.regular-expressions.info/email.html /* ****************************************************** */ ndpi_protocol_match host_match[] = { - { "s3.ll.dash.row.aiv-cdn.net", NULL, "s3\\.ll\\.dash\\.row\\.aiv-cdn\\.net", "AmazonVideo", NDPI_PROTOCOL_AMAZON_VIDEO, NDPI_PROTOCOL_CATEGORY_STREAMING, NDPI_PROTOCOL_FUN }, - { "s3-dub.cf.dash.row.aiv-cdn.net", NULL, "s3-dub\\.cf\\.dash\\.row\\.aiv-cdn\\.net", "AmazonVideo", NDPI_PROTOCOL_AMAZON_VIDEO, NDPI_PROTOCOL_CATEGORY_STREAMING, NDPI_PROTOCOL_FUN }, - { "dmqdd6hw24ucf.cloudfront.net", NULL, "dmqdd6hw24ucf\\.cloudfront\\.net", "AmazonVideo", NDPI_PROTOCOL_AMAZON_VIDEO, NDPI_PROTOCOL_CATEGORY_STREAMING, NDPI_PROTOCOL_FUN }, - { "d25xi40x97liuc.cloudfront.net", NULL, "d25xi40x97liuc\\.cloudfront\\.net", "AmazonVideo", NDPI_PROTOCOL_AMAZON_VIDEO, NDPI_PROTOCOL_CATEGORY_STREAMING, NDPI_PROTOCOL_FUN }, - { ".aiv-delivery.net", NULL, "\\.aiv-delivery\\.net", "AmazonVideo", NDPI_PROTOCOL_AMAZON_VIDEO, NDPI_PROTOCOL_CATEGORY_STREAMING, NDPI_PROTOCOL_FUN }, + { "s3.ll.dash.row.aiv-cdn.net", NULL, "s3\\.ll\\.dash\\.row\\.aiv-cdn\\.net", "AmazonVideo", NDPI_PROTOCOL_AMAZON_VIDEO, NDPI_PROTOCOL_CATEGORY_VIDEO, NDPI_PROTOCOL_FUN }, + { "s3-dub.cf.dash.row.aiv-cdn.net", NULL, "s3-dub\\.cf\\.dash\\.row\\.aiv-cdn\\.net", "AmazonVideo", NDPI_PROTOCOL_AMAZON_VIDEO, NDPI_PROTOCOL_CATEGORY_VIDEO, NDPI_PROTOCOL_FUN }, + { "dmqdd6hw24ucf.cloudfront.net", NULL, "dmqdd6hw24ucf\\.cloudfront\\.net", "AmazonVideo", NDPI_PROTOCOL_AMAZON_VIDEO, NDPI_PROTOCOL_CATEGORY_VIDEO, NDPI_PROTOCOL_FUN }, + { "d25xi40x97liuc.cloudfront.net", NULL, "d25xi40x97liuc\\.cloudfront\\.net", "AmazonVideo", NDPI_PROTOCOL_AMAZON_VIDEO, NDPI_PROTOCOL_CATEGORY_VIDEO, NDPI_PROTOCOL_FUN }, + { ".aiv-delivery.net", NULL, "\\.aiv-delivery\\.net", "AmazonVideo", NDPI_PROTOCOL_AMAZON_VIDEO, NDPI_PROTOCOL_CATEGORY_VIDEO, NDPI_PROTOCOL_FUN }, { "amazon.", NULL, NULL, "Amazon", NDPI_PROTOCOL_AMAZON, NDPI_PROTOCOL_CATEGORY_WEB, NDPI_PROTOCOL_SAFE }, { "amazon.com", NULL, "amazon" TLD, "Amazon", NDPI_PROTOCOL_AMAZON, NDPI_PROTOCOL_CATEGORY_WEB, NDPI_PROTOCOL_SAFE }, { "images-amazon.com", NULL, "images-amazon" TLD, "Amazon", NDPI_PROTOCOL_AMAZON, NDPI_PROTOCOL_CATEGORY_WEB, NDPI_PROTOCOL_ACCEPTABLE }, @@ -8070,13 +8070,13 @@ ndpi_protocol_match host_match[] = { { ".dropbox-dns.com", NULL, "\\.dropbox-dns" TLD, "DropBox", NDPI_PROTOCOL_DROPBOX, NDPI_PROTOCOL_CATEGORY_CLOUD, NDPI_PROTOCOL_ACCEPTABLE }, { "log.getdropbox.com", NULL, "log\\.getdropbox" TLD, "DropBox", NDPI_PROTOCOL_DROPBOX, NDPI_PROTOCOL_CATEGORY_CLOUD, NDPI_PROTOCOL_ACCEPTABLE }, - { ".ebay.", NULL, "\\.ebay" TLD, "eBay", NDPI_PROTOCOL_EBAY, NDPI_PROTOCOL_CATEGORY_WEB, NDPI_PROTOCOL_SAFE }, /* or FUN */ - { ".ebay.com", NULL, NULL, "eBay", NDPI_PROTOCOL_EBAY, NDPI_PROTOCOL_CATEGORY_WEB, NDPI_PROTOCOL_SAFE }, - { ".ebaystatic.com", NULL, "\\.ebaystatic" TLD, "eBay", NDPI_PROTOCOL_EBAY, NDPI_PROTOCOL_CATEGORY_WEB, NDPI_PROTOCOL_SAFE }, - { ".ebaydesc.com", NULL, "\\.ebaydesc" TLD, "eBay", NDPI_PROTOCOL_EBAY, NDPI_PROTOCOL_CATEGORY_WEB, NDPI_PROTOCOL_SAFE }, - { ".ebayrtm.com", NULL, "\\.ebayrtm" TLD, "eBay", NDPI_PROTOCOL_EBAY, NDPI_PROTOCOL_CATEGORY_WEB, NDPI_PROTOCOL_SAFE }, - { ".ebaystratus.com", NULL, "\\.ebaystratus" TLD, "eBay", NDPI_PROTOCOL_EBAY, NDPI_PROTOCOL_CATEGORY_WEB, NDPI_PROTOCOL_SAFE }, - { ".ebayimg.com", NULL, "\\.ebayimg" TLD, "eBay", NDPI_PROTOCOL_EBAY, NDPI_PROTOCOL_CATEGORY_WEB, NDPI_PROTOCOL_SAFE }, + { ".ebay.", NULL, "\\.ebay" TLD, "eBay", NDPI_PROTOCOL_EBAY, NDPI_PROTOCOL_CATEGORY_SHOPPING, NDPI_PROTOCOL_SAFE }, /* or FUN */ + { ".ebay.com", NULL, NULL, "eBay", NDPI_PROTOCOL_EBAY, NDPI_PROTOCOL_CATEGORY_SHOPPING, NDPI_PROTOCOL_SAFE }, + { ".ebaystatic.com", NULL, "\\.ebaystatic" TLD, "eBay", NDPI_PROTOCOL_EBAY, NDPI_PROTOCOL_CATEGORY_SHOPPING, NDPI_PROTOCOL_SAFE }, + { ".ebaydesc.com", NULL, "\\.ebaydesc" TLD, "eBay", NDPI_PROTOCOL_EBAY, NDPI_PROTOCOL_CATEGORY_SHOPPING, NDPI_PROTOCOL_SAFE }, + { ".ebayrtm.com", NULL, "\\.ebayrtm" TLD, "eBay", NDPI_PROTOCOL_EBAY, NDPI_PROTOCOL_CATEGORY_SHOPPING, NDPI_PROTOCOL_SAFE }, + { ".ebaystratus.com", NULL, "\\.ebaystratus" TLD, "eBay", NDPI_PROTOCOL_EBAY, NDPI_PROTOCOL_CATEGORY_SHOPPING, NDPI_PROTOCOL_SAFE }, + { ".ebayimg.com", NULL, "\\.ebayimg" TLD, "eBay", NDPI_PROTOCOL_EBAY, NDPI_PROTOCOL_CATEGORY_SHOPPING, NDPI_PROTOCOL_SAFE }, /* Detected "instagram.c10r.facebook.com". Omitted "*amazonaws.com" and "*facebook.com" CDNs e.g. "ig-telegraph-shv-04-frc3.facebook.com" */ { ".instagram.", NULL, "\\.instagram" TLD, "Instagram", NDPI_PROTOCOL_INSTAGRAM, NDPI_PROTOCOL_CATEGORY_SOCIAL_NETWORK, NDPI_PROTOCOL_FUN }, @@ -8126,8 +8126,6 @@ ndpi_protocol_match host_match[] = { Gstatic by Google (gstatic.com) */ - - /* Google Advertisements */ { ".googlesyndication.com", NULL, "\\.googlesyndication" TLD, "Google", NDPI_PROTOCOL_GOOGLE, NDPI_PROTOCOL_CATEGORY_WEB, NDPI_PROTOCOL_TRACKER_ADS }, { "googleads.", NULL, "googleads\\.", "Google", NDPI_PROTOCOL_GOOGLE, NDPI_PROTOCOL_CATEGORY_WEB, NDPI_PROTOCOL_TRACKER_ADS }, @@ -8168,16 +8166,16 @@ ndpi_protocol_match host_match[] = { { "mail.outlook.com", NULL, "mail\\.outlook" TLD, "Hotmail", NDPI_PROTOCOL_HOTMAIL, NDPI_PROTOCOL_CATEGORY_MAIL, NDPI_PROTOCOL_ACCEPTABLE }, - { ".last.fm", NULL, "\\.last\\.fm$", "LastFM", NDPI_PROTOCOL_LASTFM, NDPI_PROTOCOL_CATEGORY_STREAMING, NDPI_PROTOCOL_FUN }, + { ".last.fm", NULL, "\\.last\\.fm$", "LastFM", NDPI_PROTOCOL_LASTFM, NDPI_PROTOCOL_CATEGORY_MUSIC, NDPI_PROTOCOL_FUN }, { "msn.com", NULL, "msn" TLD, "MSN", NDPI_PROTOCOL_MSN, NDPI_PROTOCOL_CATEGORY_WEB, NDPI_PROTOCOL_SAFE }, /* News site */ - { "netflix.com", NULL, "netflix" TLD, "NetFlix", NDPI_PROTOCOL_NETFLIX, NDPI_PROTOCOL_CATEGORY_STREAMING, NDPI_PROTOCOL_FUN }, - { "nflxext.com", NULL, "nflxext" TLD, "NetFlix", NDPI_PROTOCOL_NETFLIX, NDPI_PROTOCOL_CATEGORY_STREAMING, NDPI_PROTOCOL_FUN }, - { "nflximg.com", NULL, "nflximg" TLD, "NetFlix", NDPI_PROTOCOL_NETFLIX, NDPI_PROTOCOL_CATEGORY_STREAMING, NDPI_PROTOCOL_FUN }, - { "nflximg.net", NULL, "nflximg" TLD, "NetFlix", NDPI_PROTOCOL_NETFLIX, NDPI_PROTOCOL_CATEGORY_STREAMING, NDPI_PROTOCOL_FUN }, - { "nflxvideo.net", NULL, "nflxvideo" TLD, "NetFlix", NDPI_PROTOCOL_NETFLIX, NDPI_PROTOCOL_CATEGORY_STREAMING, NDPI_PROTOCOL_FUN }, - { "nflxso.net", NULL, "nflxso" TLD, "NetFlix", NDPI_PROTOCOL_NETFLIX, NDPI_PROTOCOL_CATEGORY_STREAMING, NDPI_PROTOCOL_FUN }, + { "netflix.com", NULL, "netflix" TLD, "NetFlix", NDPI_PROTOCOL_NETFLIX, NDPI_PROTOCOL_CATEGORY_VIDEO, NDPI_PROTOCOL_FUN }, + { "nflxext.com", NULL, "nflxext" TLD, "NetFlix", NDPI_PROTOCOL_NETFLIX, NDPI_PROTOCOL_CATEGORY_VIDEO, NDPI_PROTOCOL_FUN }, + { "nflximg.com", NULL, "nflximg" TLD, "NetFlix", NDPI_PROTOCOL_NETFLIX, NDPI_PROTOCOL_CATEGORY_VIDEO, NDPI_PROTOCOL_FUN }, + { "nflximg.net", NULL, "nflximg" TLD, "NetFlix", NDPI_PROTOCOL_NETFLIX, NDPI_PROTOCOL_CATEGORY_VIDEO, NDPI_PROTOCOL_FUN }, + { "nflxvideo.net", NULL, "nflxvideo" TLD, "NetFlix", NDPI_PROTOCOL_NETFLIX, NDPI_PROTOCOL_CATEGORY_VIDEO, NDPI_PROTOCOL_FUN }, + { "nflxso.net", NULL, "nflxso" TLD, "NetFlix", NDPI_PROTOCOL_NETFLIX, NDPI_PROTOCOL_CATEGORY_VIDEO, NDPI_PROTOCOL_FUN }, { ".skype.", NULL, "\\.skype\\.", "Skype", NDPI_PROTOCOL_SKYPE, NDPI_PROTOCOL_CATEGORY_VOIP, NDPI_PROTOCOL_ACCEPTABLE }, { ".skypeassets.", NULL, "\\.skypeassets\\.", "Skype", NDPI_PROTOCOL_SKYPE, NDPI_PROTOCOL_CATEGORY_VOIP, NDPI_PROTOCOL_ACCEPTABLE }, @@ -8221,10 +8219,10 @@ ndpi_protocol_match host_match[] = { { ".ytimg.com", NULL, "\\.ytimg" TLD, "YouTube", NDPI_PROTOCOL_YOUTUBE, NDPI_PROTOCOL_CATEGORY_MEDIA, NDPI_PROTOCOL_FUN }, { "youtube-nocookie.", NULL, "youtube-nocookie" TLD, "YouTube", NDPI_PROTOCOL_YOUTUBE, NDPI_PROTOCOL_CATEGORY_MEDIA, NDPI_PROTOCOL_FUN }, - { ".vevo.com", NULL, "\\.vevo" TLD, "Vevo", NDPI_PROTOCOL_VEVO, NDPI_PROTOCOL_CATEGORY_MEDIA, NDPI_PROTOCOL_FUN }, + { ".vevo.com", NULL, "\\.vevo" TLD, "Vevo", NDPI_PROTOCOL_VEVO, NDPI_PROTOCOL_CATEGORY_MUSIC, NDPI_PROTOCOL_FUN }, - { ".spotify.", NULL, "\\.spotify" TLD, "Spotify", NDPI_PROTOCOL_SPOTIFY, NDPI_PROTOCOL_CATEGORY_MEDIA, NDPI_PROTOCOL_FUN }, - { "audio-fa.scdn.co", NULL, "audio-fa\\.scdn" TLD, "Spotify", NDPI_PROTOCOL_SPOTIFY, NDPI_PROTOCOL_CATEGORY_MEDIA, NDPI_PROTOCOL_FUN }, + { ".spotify.", NULL, "\\.spotify" TLD, "Spotify", NDPI_PROTOCOL_SPOTIFY, NDPI_PROTOCOL_CATEGORY_MUSIC, NDPI_PROTOCOL_FUN }, + { "audio-fa.scdn.co", NULL, "audio-fa\\.scdn" TLD, "Spotify", NDPI_PROTOCOL_SPOTIFY, NDPI_PROTOCOL_CATEGORY_MUSIC, NDPI_PROTOCOL_FUN }, { "edge-mqtt.facebook.com", NULL, "edge-mqtt\\.facebook" TLD, "Messenger", NDPI_PROTOCOL_MESSENGER, NDPI_PROTOCOL_CATEGORY_CHAT, NDPI_PROTOCOL_FUN }, { "messenger.com", NULL, "messenger\\.com" TLD, "Messenger", NDPI_PROTOCOL_MESSENGER, NDPI_PROTOCOL_CATEGORY_CHAT, NDPI_PROTOCOL_FUN }, @@ -8233,11 +8231,11 @@ ndpi_protocol_match host_match[] = { { ".torproject.org", NULL, "\\.torproject\\.org$", "Tor", NDPI_PROTOCOL_TOR, NDPI_PROTOCOL_CATEGORY_VPN, NDPI_PROTOCOL_POTENTIALLY_DANGEROUS }, - { ".kakao.com", NULL, "\\.kakao" TLD, "KakaoTalk", NDPI_PROTOCOL_KAKAOTALK, NDPI_PROTOCOL_CATEGORY_VOIP, NDPI_PROTOCOL_ACCEPTABLE }, + { ".kakao.com", NULL, "\\.kakao" TLD, "KakaoTalk", NDPI_PROTOCOL_KAKAOTALK, NDPI_PROTOCOL_CATEGORY_CHAT, NDPI_PROTOCOL_ACCEPTABLE }, - { "ttvnw.net", NULL, "ttvnw" TLD, "Twitch", NDPI_PROTOCOL_TWITCH, NDPI_PROTOCOL_CATEGORY_MEDIA, NDPI_PROTOCOL_FUN }, - { "static-cdn.jtvnw.net", NULL, "static-cdn\\.jtvnw" TLD, "Twitch", NDPI_PROTOCOL_TWITCH, NDPI_PROTOCOL_CATEGORY_MEDIA, NDPI_PROTOCOL_FUN }, - { "www-cdn.jtvnw.net", NULL, "www-cdn\\.jtvnw" TLD, "Twitch", NDPI_PROTOCOL_TWITCH, NDPI_PROTOCOL_CATEGORY_MEDIA, NDPI_PROTOCOL_FUN }, + { "ttvnw.net", NULL, "ttvnw" TLD, "Twitch", NDPI_PROTOCOL_TWITCH, NDPI_PROTOCOL_CATEGORY_VIDEO, NDPI_PROTOCOL_FUN }, + { "static-cdn.jtvnw.net", NULL, "static-cdn\\.jtvnw" TLD, "Twitch", NDPI_PROTOCOL_TWITCH, NDPI_PROTOCOL_CATEGORY_VIDEO, NDPI_PROTOCOL_FUN }, + { "www-cdn.jtvnw.net", NULL, "www-cdn\\.jtvnw" TLD, "Twitch", NDPI_PROTOCOL_TWITCH, NDPI_PROTOCOL_CATEGORY_VIDEO, NDPI_PROTOCOL_FUN }, { ".qq.com", NULL, "\\.qq" TLD, "QQ", NDPI_PROTOCOL_QQ, NDPI_PROTOCOL_CATEGORY_CHAT, NDPI_PROTOCOL_FUN }, { ".gtimg.com", NULL, "\\.gtimg" TLD, "QQ", NDPI_PROTOCOL_QQ, NDPI_PROTOCOL_CATEGORY_CHAT, NDPI_PROTOCOL_FUN }, @@ -8250,15 +8248,15 @@ ndpi_protocol_match host_match[] = { { ".sina.com.cn", NULL, "\\.sina\\.com\\.cn$", "Sina", NDPI_PROTOCOL_SINA, NDPI_PROTOCOL_CATEGORY_SOCIAL_NETWORK, NDPI_PROTOCOL_FUN }, /* https://support.cipafilter.com/index.php?/Knowledgebase/Article/View/117/0/snapchat---how-to-block */ - { "feelinsonice.appspot.com", NULL, "\\.appspot" TLD, "Snapchat", NDPI_PROTOCOL_SNAPCHAT, NDPI_PROTOCOL_CATEGORY_CHAT, NDPI_PROTOCOL_FUN }, - { "feelinsonice-hrd.appspot.com", NULL, "feelinsonice-hrd\\.appspot" TLD, "Snapchat", NDPI_PROTOCOL_SNAPCHAT, NDPI_PROTOCOL_CATEGORY_CHAT, NDPI_PROTOCOL_FUN }, - { "feelinsonice.com", NULL, "\\.feelsonice" TLD, "Snapchat", NDPI_PROTOCOL_SNAPCHAT, NDPI_PROTOCOL_CATEGORY_CHAT, NDPI_PROTOCOL_FUN }, - { ".snapchat.", NULL, "\\.snapchat" TLD, "Snapchat", NDPI_PROTOCOL_SNAPCHAT, NDPI_PROTOCOL_CATEGORY_CHAT, NDPI_PROTOCOL_FUN }, - { ".snapads.", NULL, "\\.snapads" TLD, "Snapchat", NDPI_PROTOCOL_SNAPCHAT, NDPI_PROTOCOL_CATEGORY_CHAT, NDPI_PROTOCOL_FUN }, + { "feelinsonice.appspot.com", NULL, "\\.appspot" TLD, "Snapchat", NDPI_PROTOCOL_SNAPCHAT, NDPI_PROTOCOL_CATEGORY_SOCIAL_NETWORK, NDPI_PROTOCOL_FUN }, + { "feelinsonice-hrd.appspot.com", NULL, "feelinsonice-hrd\\.appspot" TLD, "Snapchat", NDPI_PROTOCOL_SNAPCHAT, NDPI_PROTOCOL_CATEGORY_SOCIAL_NETWORK, NDPI_PROTOCOL_FUN }, + { "feelinsonice.com", NULL, "\\.feelsonice" TLD, "Snapchat", NDPI_PROTOCOL_SNAPCHAT, NDPI_PROTOCOL_CATEGORY_SOCIAL_NETWORK, NDPI_PROTOCOL_FUN }, + { ".snapchat.", NULL, "\\.snapchat" TLD, "Snapchat", NDPI_PROTOCOL_SNAPCHAT, NDPI_PROTOCOL_CATEGORY_SOCIAL_NETWORK, NDPI_PROTOCOL_FUN }, + { ".snapads.", NULL, "\\.snapads" TLD, "Snapchat", NDPI_PROTOCOL_SNAPCHAT, NDPI_PROTOCOL_CATEGORY_SOCIAL_NETWORK, NDPI_PROTOCOL_FUN }, { ".waze.com", NULL, "\\.waze" TLD, "Waze", NDPI_PROTOCOL_WAZE, NDPI_PROTOCOL_CATEGORY_WEB, NDPI_PROTOCOL_ACCEPTABLE }, - { ".deezer.com", NULL, "\\.deezer" TLD, "Deezer", NDPI_PROTOCOL_DEEZER, NDPI_PROTOCOL_CATEGORY_STREAMING, NDPI_PROTOCOL_FUN }, + { ".deezer.com", NULL, "\\.deezer" TLD, "Deezer", NDPI_PROTOCOL_DEEZER, NDPI_PROTOCOL_CATEGORY_MUSIC, NDPI_PROTOCOL_FUN }, { ".microsoft.com", NULL, "\\.microsoft" TLD, "Microsoft", NDPI_PROTOCOL_MICROSOFT, NDPI_PROTOCOL_CATEGORY_WEB, NDPI_PROTOCOL_SAFE }, { "i-msdn.sec.s-msft.com", NULL, "i-msdn.sec\\.s-msft" TLD, "Microsoft", NDPI_PROTOCOL_MICROSOFT, NDPI_PROTOCOL_CATEGORY_SYSTEM_OS, NDPI_PROTOCOL_ACCEPTABLE }, @@ -8291,9 +8289,9 @@ ndpi_protocol_match host_match[] = { { ".ocs.fr", NULL, NULL, "OCS", NDPI_PROTOCOL_OCS, NDPI_PROTOCOL_CATEGORY_MEDIA, NDPI_PROTOCOL_FUN }, { ".labgency.ws", NULL, ".labgency" TLD, "OCS", NDPI_PROTOCOL_OCS, NDPI_PROTOCOL_CATEGORY_MEDIA, NDPI_PROTOCOL_FUN }, - { ".iflix.com", NULL, "\\.iflix" TLD, "IFLIX", NDPI_PROTOCOL_IFLIX, NDPI_PROTOCOL_CATEGORY_MEDIA, NDPI_PROTOCOL_FUN }, - { ".app.iflixcorp.com", NULL, "\\.app\\.iflixcorp" TLD, "IFLIX", NDPI_PROTOCOL_IFLIX, NDPI_PROTOCOL_CATEGORY_MEDIA, NDPI_PROTOCOL_FUN }, - { ".images.iflixassets.com", NULL, "\\.images\\.iflixassets" TLD, "IFLIX", NDPI_PROTOCOL_IFLIX, NDPI_PROTOCOL_CATEGORY_MEDIA, NDPI_PROTOCOL_FUN }, + { ".iflix.com", NULL, "\\.iflix" TLD, "IFLIX", NDPI_PROTOCOL_IFLIX, NDPI_PROTOCOL_CATEGORY_VIDEO, NDPI_PROTOCOL_FUN }, + { ".app.iflixcorp.com", NULL, "\\.app\\.iflixcorp" TLD, "IFLIX", NDPI_PROTOCOL_IFLIX, NDPI_PROTOCOL_CATEGORY_VIDEO, NDPI_PROTOCOL_FUN }, + { ".images.iflixassets.com", NULL, "\\.images\\.iflixassets" TLD, "IFLIX", NDPI_PROTOCOL_IFLIX, NDPI_PROTOCOL_CATEGORY_VIDEO, NDPI_PROTOCOL_FUN }, { "crl.microsoft.com", NULL, "crl\\.microsoft" TLD, "Office365", NDPI_PROTOCOL_OFFICE_365, NDPI_PROTOCOL_CATEGORY_COLLABORATIVE, NDPI_PROTOCOL_ACCEPTABLE }, { "evsecure-ocsp.verisign.com", NULL, "evsecure-ocsp\\.verisign" TLD,"Office365", NDPI_PROTOCOL_OFFICE_365, NDPI_PROTOCOL_CATEGORY_COLLABORATIVE, NDPI_PROTOCOL_ACCEPTABLE }, @@ -8369,9 +8367,9 @@ ndpi_protocol_match host_match[] = { { ".linkedin.com", NULL, "\\.linkedin" TLD, "LinkedIn", NDPI_PROTOCOL_LINKEDIN, NDPI_PROTOCOL_CATEGORY_SOCIAL_NETWORK, NDPI_PROTOCOL_FUN }, { ".licdn.com", NULL, "\\.licdn" TLD, "LinkedIn", NDPI_PROTOCOL_LINKEDIN, NDPI_PROTOCOL_CATEGORY_SOCIAL_NETWORK, NDPI_PROTOCOL_FUN }, - { ".sndcdn.com", NULL, "\\.sndcdn" TLD, "SoundCloud", NDPI_PROTOCOL_SOUNDCLOUD, NDPI_PROTOCOL_CATEGORY_STREAMING, NDPI_PROTOCOL_FUN }, - { ".soundcloud.com", NULL, "\\.soundcloud" TLD, "SoundCloud", NDPI_PROTOCOL_SOUNDCLOUD, NDPI_PROTOCOL_CATEGORY_STREAMING, NDPI_PROTOCOL_FUN }, - { "getrockerbox.com", NULL, "getrockerbox" TLD, "SoundCloud", NDPI_PROTOCOL_SOUNDCLOUD, NDPI_PROTOCOL_CATEGORY_STREAMING, NDPI_PROTOCOL_FUN }, + { ".sndcdn.com", NULL, "\\.sndcdn" TLD, "SoundCloud", NDPI_PROTOCOL_SOUNDCLOUD, NDPI_PROTOCOL_CATEGORY_MUSIC, NDPI_PROTOCOL_FUN }, + { ".soundcloud.com", NULL, "\\.soundcloud" TLD, "SoundCloud", NDPI_PROTOCOL_SOUNDCLOUD, NDPI_PROTOCOL_CATEGORY_MUSIC, NDPI_PROTOCOL_FUN }, + { "getrockerbox.com", NULL, "getrockerbox" TLD, "SoundCloud", NDPI_PROTOCOL_SOUNDCLOUD, NDPI_PROTOCOL_CATEGORY_MUSIC, NDPI_PROTOCOL_FUN }, { "web.telegram.org", NULL, "web\\.telegram" TLD, "Telegram", NDPI_PROTOCOL_TELEGRAM, NDPI_PROTOCOL_CATEGORY_CHAT, NDPI_PROTOCOL_ACCEPTABLE }, { "tdesktop.com", NULL, "tdesktop" TLD, "Telegram", NDPI_PROTOCOL_TELEGRAM, NDPI_PROTOCOL_CATEGORY_CHAT, NDPI_PROTOCOL_ACCEPTABLE }, @@ -8414,8 +8412,8 @@ ndpi_protocol_match host_match[] = { { ".playercdn.net", NULL, "\\.playercdn" TLD, "RapidVideo", NDPI_PROTOCOL_GENERIC, NDPI_PROTOCOL_CATEGORY_STREAMING, NDPI_PROTOCOL_FUN }, /* showmax.com video streaming */ - { "showmax.com", NULL, "showmax" TLD, "Showmax", NDPI_PROTOCOL_GENERIC, NDPI_PROTOCOL_CATEGORY_MEDIA, NDPI_PROTOCOL_FUN }, - { "showmax.akamaized.net", NULL, "showmax\\.akamaized" TLD, "Showmax", NDPI_PROTOCOL_GENERIC, NDPI_PROTOCOL_CATEGORY_MEDIA, NDPI_PROTOCOL_FUN }, + { "showmax.com", NULL, "showmax" TLD, "Showmax", NDPI_PROTOCOL_GENERIC, NDPI_PROTOCOL_CATEGORY_VIDEO, NDPI_PROTOCOL_FUN }, + { "showmax.akamaized.net", NULL, "showmax\\.akamaized" TLD, "Showmax", NDPI_PROTOCOL_GENERIC, NDPI_PROTOCOL_CATEGORY_VIDEO, NDPI_PROTOCOL_FUN }, { "snapcraft.io", NULL, "snapcraft\\.io" TLD, "UbuntuONE", NDPI_PROTOCOL_UBUNTUONE, NDPI_PROTOCOL_CATEGORY_CLOUD, NDPI_PROTOCOL_ACCEPTABLE }, { "ubuntu.com", NULL, "ubuntu\\.com" TLD, "UbuntuONE", NDPI_PROTOCOL_UBUNTUONE, NDPI_PROTOCOL_CATEGORY_CLOUD, NDPI_PROTOCOL_ACCEPTABLE }, @@ -8461,7 +8459,7 @@ ndpi_protocol_match content_match[] = { { "audio/x-wav", NULL, NULL, NULL, NDPI_CONTENT_CATEGORY_WINDOWSMEDIA, NDPI_PROTOCOL_CATEGORY_MEDIA, NDPI_PROTOCOL_FUN }, { "application/vnd.ms.wms-hdr.asfv1", NULL, NULL, NULL, NDPI_CONTENT_CATEGORY_WINDOWSMEDIA, NDPI_PROTOCOL_CATEGORY_MEDIA, NDPI_PROTOCOL_FUN }, { "NSPlayer/", NULL, NULL, NULL, NDPI_CONTENT_CATEGORY_WINDOWSMEDIA, NDPI_PROTOCOL_CATEGORY_MEDIA, NDPI_PROTOCOL_FUN }, - { "Xbox Live Client/", NULL, NULL, NULL, NDPI_PROTOCOL_XBOX, NDPI_PROTOCOL_CATEGORY_MEDIA, NDPI_PROTOCOL_FUN }, + { "Xbox Live Client/", NULL, NULL, NULL, NDPI_PROTOCOL_XBOX, NDPI_PROTOCOL_CATEGORY_GAME, NDPI_PROTOCOL_FUN }, { "Windows-Update-Agent", NULL, NULL, NULL, NDPI_PROTOCOL_WINDOWS_UPDATE, NDPI_PROTOCOL_CATEGORY_MEDIA, NDPI_PROTOCOL_ACCEPTABLE }, { "audio/webm", NULL, NULL, NULL, NDPI_CONTENT_CATEGORY_WEBM, NDPI_PROTOCOL_CATEGORY_MEDIA, NDPI_PROTOCOL_FUN }, { "video/webm", NULL, NULL, NULL, NDPI_CONTENT_CATEGORY_WEBM, NDPI_PROTOCOL_CATEGORY_MEDIA, NDPI_PROTOCOL_FUN }, diff --git a/src/lib/ndpi_main.c b/src/lib/ndpi_main.c index 62914efbe..01b25a1b8 100644 --- a/src/lib/ndpi_main.c +++ b/src/lib/ndpi_main.c @@ -1155,7 +1155,7 @@ static void ndpi_init_protocol_defaults(struct ndpi_detection_module_struct *ndp ndpi_build_default_ports(ports_b, 902, 903, 0, 0, 0) /* UDP */); ndpi_set_proto_defaults(ndpi_mod, NDPI_PROTOCOL_SAFE, NDPI_PROTOCOL_FBZERO, 0 /* can_have_a_subprotocol */, no_master, - no_master, "FacebookZero", NDPI_PROTOCOL_CATEGORY_WEB, + no_master, "FacebookZero", NDPI_PROTOCOL_CATEGORY_SOCIAL_NETWORK, ndpi_build_default_ports(ports_a, 443, 0, 0, 0, 0) /* TCP */, ndpi_build_default_ports(ports_b, 0, 0, 0, 0, 0) /* UDP */); ndpi_set_proto_defaults(ndpi_mod, NDPI_PROTOCOL_POTENTIALLY_DANGEROUS, NDPI_PROTOCOL_KONTIKI, @@ -1268,9 +1268,9 @@ static void ndpi_init_protocol_defaults(struct ndpi_detection_module_struct *ndp no_master, "Free", NDPI_PROTOCOL_CATEGORY_CUSTOM_1 /* dummy */, ndpi_build_default_ports(ports_a, 0, 0, 0, 0, 0) /* TCP */, ndpi_build_default_ports(ports_b, 0, 0, 0, 0, 0) /* UDP */); - ndpi_set_proto_defaults(ndpi_mod, NDPI_PROTOCOL_FUN, NDPI_PROTOCOL_VIDTO, + ndpi_set_proto_defaults(ndpi_mod, NDPI_PROTOCOL_FUN, NDPI_PROTOCOL_PPSTREAM, 0 /* can_have_a_subprotocol */, no_master, - no_master, "PPStream", NDPI_PROTOCOL_CATEGORY_MEDIA, + no_master, "PPStream", NDPI_PROTOCOL_CATEGORY_VIDEO, ndpi_build_default_ports(ports_a, 0, 0, 0, 0, 0) /* TCP */, ndpi_build_default_ports(ports_b, 0, 0, 0, 0, 0) /* UDP */); @@ -1306,27 +1306,27 @@ static void ndpi_init_protocol_defaults(struct ndpi_detection_module_struct *ndp ndpi_build_default_ports(ports_b, 0, 0, 0, 0, 0) /* UDP */); ndpi_set_proto_defaults(ndpi_mod, NDPI_PROTOCOL_FUN, NDPI_PROTOCOL_ZATTOO, 0 /* can_have_a_subprotocol */, no_master, - no_master, "Zattoo", NDPI_PROTOCOL_CATEGORY_MEDIA, + no_master, "Zattoo", NDPI_PROTOCOL_CATEGORY_VIDEO, ndpi_build_default_ports(ports_a, 0, 0, 0, 0, 0) /* TCP */, ndpi_build_default_ports(ports_b, 0, 0, 0, 0, 0) /* UDP */); ndpi_set_proto_defaults(ndpi_mod, NDPI_PROTOCOL_FUN, NDPI_PROTOCOL_SHOUTCAST, 0 /* can_have_a_subprotocol */, no_master, - no_master, "ShoutCast", NDPI_PROTOCOL_CATEGORY_MEDIA, + no_master, "ShoutCast", NDPI_PROTOCOL_CATEGORY_MUSIC, ndpi_build_default_ports(ports_a, 0, 0, 0, 0, 0) /* TCP */, ndpi_build_default_ports(ports_b, 0, 0, 0, 0, 0) /* UDP */); ndpi_set_proto_defaults(ndpi_mod, NDPI_PROTOCOL_FUN, NDPI_PROTOCOL_SOPCAST, 0 /* can_have_a_subprotocol */, no_master, - no_master, "Sopcast", NDPI_PROTOCOL_CATEGORY_MEDIA, + no_master, "Sopcast", NDPI_PROTOCOL_CATEGORY_VIDEO, ndpi_build_default_ports(ports_a, 0, 0, 0, 0, 0) /* TCP */, ndpi_build_default_ports(ports_b, 0, 0, 0, 0, 0) /* UDP */); ndpi_set_proto_defaults(ndpi_mod, NDPI_PROTOCOL_FUN, NDPI_PROTOCOL_TVANTS, 0 /* can_have_a_subprotocol */, no_master, - no_master, "Tvants", NDPI_PROTOCOL_CATEGORY_MEDIA, + no_master, "Tvants", NDPI_PROTOCOL_CATEGORY_VIDEO, ndpi_build_default_ports(ports_a, 0, 0, 0, 0, 0) /* TCP */, ndpi_build_default_ports(ports_b, 0, 0, 0, 0, 0) /* UDP */); ndpi_set_proto_defaults(ndpi_mod, NDPI_PROTOCOL_FUN, NDPI_PROTOCOL_TVUPLAYER, 0 /* can_have_a_subprotocol */, no_master, - no_master, "TVUplayer", NDPI_PROTOCOL_CATEGORY_MEDIA, + no_master, "TVUplayer", NDPI_PROTOCOL_CATEGORY_VIDEO, ndpi_build_default_ports(ports_a, 0, 0, 0, 0, 0) /* TCP */, ndpi_build_default_ports(ports_b, 0, 0, 0, 0, 0) /* UDP */); ndpi_set_proto_defaults(ndpi_mod, NDPI_PROTOCOL_ACCEPTABLE, NDPI_PROTOCOL_HTTP_DOWNLOAD, @@ -1336,7 +1336,7 @@ static void ndpi_init_protocol_defaults(struct ndpi_detection_module_struct *ndp ndpi_build_default_ports(ports_b, 0, 0, 0, 0, 0) /* UDP */); ndpi_set_proto_defaults(ndpi_mod, NDPI_PROTOCOL_FUN, NDPI_PROTOCOL_QQLIVE, 0 /* can_have_a_subprotocol */, no_master, - no_master, "QQLive", NDPI_PROTOCOL_CATEGORY_MEDIA, + no_master, "QQLive", NDPI_PROTOCOL_CATEGORY_VIDEO, ndpi_build_default_ports(ports_a, 0, 0, 0, 0, 0) /* TCP */, ndpi_build_default_ports(ports_b, 0, 0, 0, 0, 0) /* UDP */); ndpi_set_proto_defaults(ndpi_mod, NDPI_PROTOCOL_FUN, NDPI_PROTOCOL_THUNDER, @@ -1784,7 +1784,7 @@ static void ndpi_init_protocol_defaults(struct ndpi_detection_module_struct *ndp ndpi_build_default_ports(ports_b, 10000, 0, 0, 0, 0) /* UDP */); ndpi_set_proto_defaults(ndpi_mod, NDPI_PROTOCOL_ACCEPTABLE, NDPI_PROTOCOL_TEAMSPEAK, 0 /* can_have_a_subprotocol */, no_master, - no_master, "TeamSpeak", NDPI_PROTOCOL_CATEGORY_CHAT, + no_master, "TeamSpeak", NDPI_PROTOCOL_CATEGORY_VOIP, ndpi_build_default_ports(ports_a, 0, 0, 0, 0, 0) /* TCP */, ndpi_build_default_ports(ports_b, 0, 0, 0, 0, 0) /* UDP */); ndpi_set_proto_defaults(ndpi_mod, NDPI_PROTOCOL_ACCEPTABLE, NDPI_PROTOCOL_SKINNY, @@ -1879,7 +1879,7 @@ static void ndpi_init_protocol_defaults(struct ndpi_detection_module_struct *ndp ndpi_build_default_ports(ports_b, 10001, 0, 0, 0, 0)); /* UDP */ ndpi_set_proto_defaults(ndpi_mod, NDPI_PROTOCOL_ACCEPTABLE, NDPI_PROTOCOL_VIBER, 0 /* can_have_a_subprotocol */, no_master, - no_master, "Viber", NDPI_PROTOCOL_CATEGORY_CHAT, + no_master, "Viber", NDPI_PROTOCOL_CATEGORY_VOIP, ndpi_build_default_ports(ports_a, 7985, 5242, 5243, 4244, 0), /* TCP */ ndpi_build_default_ports(ports_b, 7985, 7987, 5242, 5243, 4244)); /* UDP */ ndpi_set_proto_defaults(ndpi_mod, NDPI_PROTOCOL_SAFE, NDPI_PROTOCOL_COAP, @@ -1914,7 +1914,7 @@ static void ndpi_init_protocol_defaults(struct ndpi_detection_module_struct *ndp ndpi_build_default_ports(ports_b, 0, 0, 0, 0, 0)); /* UDP */ ndpi_set_proto_defaults(ndpi_mod, NDPI_PROTOCOL_ACCEPTABLE, NDPI_PROTOCOL_HANGOUT, 0 /* can_have_a_subprotocol */, no_master, - no_master, "GoogleHangout", NDPI_PROTOCOL_CATEGORY_CHAT, + no_master, "GoogleHangout", NDPI_PROTOCOL_CATEGORY_VOIP, ndpi_build_default_ports(ports_a, 0, 0, 0, 0, 0) /* TCP */, ndpi_build_default_ports(ports_b, 0, 0, 0, 0, 0) /* UDP */); ndpi_set_proto_defaults(ndpi_mod, NDPI_PROTOCOL_ACCEPTABLE, NDPI_PROTOCOL_BJNP, @@ -5684,11 +5684,11 @@ static const char* categories[] = { "RealMedia", "WindowsMedia", "Webm", /* 32 */ - "", - "", - "", - "", - "", + "Music", + "Video", + "Shopping", + "Productivity", + "FileSharing", "", "", "", -- cgit v1.2.3 From fde43804228d03911aed51076be6f4e4ab4fc0fa Mon Sep 17 00:00:00 2001 From: Luca Date: Fri, 8 Mar 2019 20:17:11 +0100 Subject: Proto cleanup --- src/include/ndpi_protocol_ids.h | 2 +- src/lib/ndpi_content_match.c.inc | 7 ------- src/lib/ndpi_main.c | 5 +++++ src/lib/protocols/ssl.c | 10 +++++----- 4 files changed, 11 insertions(+), 13 deletions(-) (limited to 'src/lib') diff --git a/src/include/ndpi_protocol_ids.h b/src/include/ndpi_protocol_ids.h index 1d9c1d428..7e2f55711 100644 --- a/src/include/ndpi_protocol_ids.h +++ b/src/include/ndpi_protocol_ids.h @@ -242,7 +242,7 @@ typedef enum { NDPI_PROTOCOL_GITHUB = 203, NDPI_PROTOCOL_BJNP = 204, NDPI_PROTOCOL_FREE_205 = 205, /* Free */ - NDPI_PROTOCOL_VIDTO = 206, + NDPI_PROTOCOL_FREE_206 = 206, /* Free */ NDPI_PROTOCOL_SMPP = 207, /* Damir Franusic */ NDPI_PROTOCOL_DNSCRYPT = 208, NDPI_PROTOCOL_TINC = 209, /* William Guglielmo */ diff --git a/src/lib/ndpi_content_match.c.inc b/src/lib/ndpi_content_match.c.inc index 8f7fc2337..c71913081 100644 --- a/src/lib/ndpi_content_match.c.inc +++ b/src/lib/ndpi_content_match.c.inc @@ -7947,13 +7947,6 @@ static ndpi_network host_protocol_list[] = { { 0xD040C900 /* 208.64.201.0/22 */, 22, NDPI_PROTOCOL_STEAM }, { 0xD04EA400 /* 208.78.164.0/22 */, 22, NDPI_PROTOCOL_STEAM }, - /* - VidTO - */ - { 0x51111030 /* 81.17.16.48/32 */, 32, NDPI_PROTOCOL_VIDTO }, - { 0x5fb7329d /* 95.183.50.157/32 */, 32, NDPI_PROTOCOL_VIDTO }, - { 0x577824f2 /* 87.120.36.242/32 */, 32, NDPI_PROTOCOL_VIDTO }, - { 0x0, 0, 0 } }; diff --git a/src/lib/ndpi_main.c b/src/lib/ndpi_main.c index 936f47dac..b8cdf410b 100644 --- a/src/lib/ndpi_main.c +++ b/src/lib/ndpi_main.c @@ -1263,6 +1263,11 @@ static void ndpi_init_protocol_defaults(struct ndpi_detection_module_struct *ndp no_master, "Free", NDPI_PROTOCOL_CATEGORY_CUSTOM_1 /* dummy */, ndpi_build_default_ports(ports_a, 0, 0, 0, 0, 0) /* TCP */, ndpi_build_default_ports(ports_b, 0, 0, 0, 0, 0) /* UDP */); + ndpi_set_proto_defaults(ndpi_mod, NDPI_PROTOCOL_FUN, NDPI_PROTOCOL_FREE_206, + 0 /* can_have_a_subprotocol */, no_master, + no_master, "Free", NDPI_PROTOCOL_CATEGORY_CUSTOM_1 /* dummy */, + ndpi_build_default_ports(ports_a, 0, 0, 0, 0, 0) /* TCP */, + ndpi_build_default_ports(ports_b, 0, 0, 0, 0, 0) /* UDP */); ndpi_set_proto_defaults(ndpi_mod, NDPI_PROTOCOL_FUN, NDPI_PROTOCOL_PPSTREAM, 0 /* can_have_a_subprotocol */, no_master, no_master, "PPStream", NDPI_PROTOCOL_CATEGORY_VIDEO, diff --git a/src/lib/protocols/ssl.c b/src/lib/protocols/ssl.c index 4651b358f..ff6b47a0f 100644 --- a/src/lib/protocols/ssl.c +++ b/src/lib/protocols/ssl.c @@ -27,7 +27,8 @@ #include "ndpi_api.h" -// #define CERTIFICATE_DEBUG 1 +//#define CERTIFICATE_DEBUG 1 + #define NDPI_MAX_SSL_REQUEST_SIZE 10000 /* Skype.c */ @@ -150,9 +151,9 @@ int getSSLcertificate(struct ndpi_detection_module_struct *ndpi_struct, #ifdef CERTIFICATE_DEBUG { - static u_int8_t id = 0; + u_int16_t ssl_version = (packet->payload[1] << 8) + packet->payload[2]; - NDPI_LOG_DBG2(ndpi_struct,"-> [%u] %02X\n", ++id, packet->payload[0] & 0xFF); + printf("SSL [version: %u]\n", ssl_version); } #endif @@ -232,8 +233,7 @@ int getSSLcertificate(struct ndpi_detection_module_struct *ndpi_struct, } } else if(handshake_protocol == 0x01 /* Client Hello */) { u_int offset, base_offset = 43; - if (base_offset + 2 <= packet->payload_packet_len) - { + if (base_offset + 2 <= packet->payload_packet_len) { u_int16_t session_id_len = packet->payload[base_offset]; if((session_id_len+base_offset+2) <= total_len) { -- cgit v1.2.3 From 89d548f9d30a006caaf304a962f4cc4cf8cdf6c1 Mon Sep 17 00:00:00 2001 From: Luca Deri Date: Sun, 10 Mar 2019 21:18:19 +0100 Subject: Removed categories based on file content --- src/include/ndpi_typedefs.h | 10 -- src/lib/ndpi_content_match.c.inc | 57 --------- src/lib/ndpi_main.c | 6 - src/lib/protocols/http.c | 257 +-------------------------------------- 4 files changed, 2 insertions(+), 328 deletions(-) (limited to 'src/lib') diff --git a/src/include/ndpi_typedefs.h b/src/include/ndpi_typedefs.h index 917aa6827..39995d662 100644 --- a/src/include/ndpi_typedefs.h +++ b/src/include/ndpi_typedefs.h @@ -791,16 +791,6 @@ typedef enum { NDPI_PROTOCOL_CATEGORY_CUSTOM_4, /* User custom category 4 */ NDPI_PROTOCOL_CATEGORY_CUSTOM_5, /* User custom category 5 */ - /* Payload Content */ - NDPI_CONTENT_CATEGORY_AVI, - NDPI_CONTENT_CATEGORY_FLASH, - NDPI_CONTENT_CATEGORY_OGG, - NDPI_CONTENT_CATEGORY_MPEG, - NDPI_CONTENT_CATEGORY_QUICKTIME, - NDPI_CONTENT_CATEGORY_REALMEDIA, - NDPI_CONTENT_CATEGORY_WINDOWSMEDIA, - NDPI_CONTENT_CATEGORY_WEBM, - /* Further categories... */ NDPI_PROTOCOL_CATEGORY_MUSIC, NDPI_PROTOCOL_CATEGORY_VIDEO, diff --git a/src/lib/ndpi_content_match.c.inc b/src/lib/ndpi_content_match.c.inc index c71913081..3261618b9 100644 --- a/src/lib/ndpi_content_match.c.inc +++ b/src/lib/ndpi_content_match.c.inc @@ -8418,63 +8418,6 @@ ndpi_protocol_match host_match[] = { { NULL, NULL, NULL, 0 } }; - -/* - Mime-type content match match -*/ -ndpi_protocol_match content_match[] = { - { "audio/mpeg", NULL, NULL, NULL, NDPI_CONTENT_CATEGORY_MPEG, NDPI_PROTOCOL_CATEGORY_MEDIA, NDPI_PROTOCOL_FUN }, - { "audio/x-mpeg", NULL, NULL, NULL, NDPI_CONTENT_CATEGORY_MPEG, NDPI_PROTOCOL_CATEGORY_MEDIA, NDPI_PROTOCOL_FUN }, - { "audio/mpeg3", NULL, NULL, NULL, NDPI_CONTENT_CATEGORY_MPEG, NDPI_PROTOCOL_CATEGORY_MEDIA, NDPI_PROTOCOL_FUN }, - { "audio/mp4a", NULL, NULL, NULL, NDPI_CONTENT_CATEGORY_MPEG, NDPI_PROTOCOL_CATEGORY_MEDIA, NDPI_PROTOCOL_FUN }, - { "video/mpeg", NULL, NULL, NULL, NDPI_CONTENT_CATEGORY_MPEG, NDPI_PROTOCOL_CATEGORY_MEDIA, NDPI_PROTOCOL_FUN }, - { "video/nsv", NULL, NULL, NULL, NDPI_CONTENT_CATEGORY_MPEG, NDPI_PROTOCOL_CATEGORY_MEDIA, NDPI_PROTOCOL_FUN }, - { "misc/ultravox", NULL, NULL, NULL, NDPI_CONTENT_CATEGORY_MPEG, NDPI_PROTOCOL_CATEGORY_MEDIA, NDPI_PROTOCOL_FUN }, - { "audio/ogg", NULL, NULL, NULL, NDPI_CONTENT_CATEGORY_OGG, NDPI_PROTOCOL_CATEGORY_MEDIA, NDPI_PROTOCOL_FUN }, - { "video/ogg", NULL, NULL, NULL, NDPI_CONTENT_CATEGORY_OGG, NDPI_PROTOCOL_CATEGORY_MEDIA, NDPI_PROTOCOL_FUN }, - { "application/ogg", NULL, NULL, NULL, NDPI_CONTENT_CATEGORY_OGG, NDPI_PROTOCOL_CATEGORY_MEDIA, NDPI_PROTOCOL_FUN }, - { ".adobe.", NULL, NULL, NULL, NDPI_CONTENT_CATEGORY_FLASH, NDPI_PROTOCOL_CATEGORY_MEDIA, NDPI_PROTOCOL_FUN }, - { "video/flv", NULL, NULL, NULL, NDPI_CONTENT_CATEGORY_FLASH, NDPI_PROTOCOL_CATEGORY_MEDIA, NDPI_PROTOCOL_FUN }, - { "video/x-flv", NULL, NULL, NULL, NDPI_CONTENT_CATEGORY_FLASH, NDPI_PROTOCOL_CATEGORY_MEDIA, NDPI_PROTOCOL_FUN }, - { "application/x-fcs", NULL, NULL, NULL, NDPI_CONTENT_CATEGORY_FLASH, NDPI_PROTOCOL_CATEGORY_MEDIA, NDPI_PROTOCOL_FUN }, - { "application/x-shockwave-flash",NULL, NULL, NULL, NDPI_CONTENT_CATEGORY_FLASH, NDPI_PROTOCOL_CATEGORY_MEDIA, NDPI_PROTOCOL_ACCEPTABLE }, - { "video/flash", NULL, NULL, NULL, NDPI_CONTENT_CATEGORY_FLASH, NDPI_PROTOCOL_CATEGORY_MEDIA, NDPI_PROTOCOL_FUN }, - { "application/flv", NULL, NULL, NULL, NDPI_CONTENT_CATEGORY_FLASH, NDPI_PROTOCOL_CATEGORY_MEDIA, NDPI_PROTOCOL_FUN }, - { "flv-application/octet-stream", NULL, NULL, NULL, NDPI_CONTENT_CATEGORY_FLASH, NDPI_PROTOCOL_CATEGORY_MEDIA, NDPI_PROTOCOL_FUN }, - { "application/futuresplash", NULL, NULL, NULL, NDPI_CONTENT_CATEGORY_FLASH, NDPI_PROTOCOL_CATEGORY_MEDIA, NDPI_PROTOCOL_FUN }, - { "video/quicktime", NULL, NULL, NULL, NDPI_CONTENT_CATEGORY_QUICKTIME, NDPI_PROTOCOL_CATEGORY_MEDIA, NDPI_PROTOCOL_FUN }, - { "video/mp4", NULL, NULL, NULL, NDPI_CONTENT_CATEGORY_QUICKTIME, NDPI_PROTOCOL_CATEGORY_MEDIA, NDPI_PROTOCOL_FUN }, - { "video/x-m4v", NULL, NULL, NULL, NDPI_CONTENT_CATEGORY_QUICKTIME, NDPI_PROTOCOL_CATEGORY_MEDIA, NDPI_PROTOCOL_FUN }, - { "audio/x-pn-realaudio", NULL, NULL, NULL, NDPI_CONTENT_CATEGORY_REALMEDIA, NDPI_PROTOCOL_CATEGORY_MEDIA, NDPI_PROTOCOL_FUN }, - { "application/vnd.rn-realmedia", NULL, NULL, NULL, NDPI_CONTENT_CATEGORY_REALMEDIA, NDPI_PROTOCOL_CATEGORY_MEDIA, NDPI_PROTOCOL_FUN }, - { "video/x-ms-", NULL, NULL, NULL, NDPI_CONTENT_CATEGORY_WINDOWSMEDIA, NDPI_PROTOCOL_CATEGORY_MEDIA, NDPI_PROTOCOL_FUN }, - { "asf", NULL, NULL, NULL, NDPI_CONTENT_CATEGORY_WINDOWSMEDIA, NDPI_PROTOCOL_CATEGORY_MEDIA, NDPI_PROTOCOL_FUN }, - { "asx", NULL, NULL, NULL, NDPI_CONTENT_CATEGORY_WINDOWSMEDIA, NDPI_PROTOCOL_CATEGORY_MEDIA, NDPI_PROTOCOL_FUN }, - { "video/x-msvideo", NULL, NULL, NULL, NDPI_CONTENT_CATEGORY_WINDOWSMEDIA, NDPI_PROTOCOL_CATEGORY_MEDIA, NDPI_PROTOCOL_FUN }, - { "audio/x-wav", NULL, NULL, NULL, NDPI_CONTENT_CATEGORY_WINDOWSMEDIA, NDPI_PROTOCOL_CATEGORY_MEDIA, NDPI_PROTOCOL_FUN }, - { "application/vnd.ms.wms-hdr.asfv1", NULL, NULL, NULL, NDPI_CONTENT_CATEGORY_WINDOWSMEDIA, NDPI_PROTOCOL_CATEGORY_MEDIA, NDPI_PROTOCOL_FUN }, - { "NSPlayer/", NULL, NULL, NULL, NDPI_CONTENT_CATEGORY_WINDOWSMEDIA, NDPI_PROTOCOL_CATEGORY_MEDIA, NDPI_PROTOCOL_FUN }, - { "Xbox Live Client/", NULL, NULL, NULL, NDPI_PROTOCOL_XBOX, NDPI_PROTOCOL_CATEGORY_GAME, NDPI_PROTOCOL_FUN }, - { "Windows-Update-Agent", NULL, NULL, NULL, NDPI_PROTOCOL_WINDOWS_UPDATE, NDPI_PROTOCOL_CATEGORY_MEDIA, NDPI_PROTOCOL_ACCEPTABLE }, - { "audio/webm", NULL, NULL, NULL, NDPI_CONTENT_CATEGORY_WEBM, NDPI_PROTOCOL_CATEGORY_MEDIA, NDPI_PROTOCOL_FUN }, - { "video/webm", NULL, NULL, NULL, NDPI_CONTENT_CATEGORY_WEBM, NDPI_PROTOCOL_CATEGORY_MEDIA, NDPI_PROTOCOL_FUN }, - { "application/x-rtsp-tunnelled", NULL, NULL, NULL, NDPI_PROTOCOL_RTSP, NDPI_PROTOCOL_CATEGORY_MEDIA, NDPI_PROTOCOL_FUN }, - { "application/vnd.apple.mpegurl",NULL, NULL, NULL, NDPI_CONTENT_CATEGORY_MPEG, NDPI_PROTOCOL_CATEGORY_MEDIA, NDPI_PROTOCOL_FUN }, - { "application/x-tar", NULL, NULL, NULL, NDPI_PROTOCOL_HTTP_DOWNLOAD, NDPI_PROTOCOL_CATEGORY_DOWNLOAD_FT, NDPI_PROTOCOL_ACCEPTABLE }, - { "application/octet-stream", NULL, NULL, NULL, NDPI_PROTOCOL_HTTP_DOWNLOAD, NDPI_PROTOCOL_CATEGORY_DOWNLOAD_FT, NDPI_PROTOCOL_ACCEPTABLE }, - { "application/mac-binary", NULL, NULL, NULL, NDPI_PROTOCOL_HTTP_DOWNLOAD, NDPI_PROTOCOL_CATEGORY_DOWNLOAD_FT, NDPI_PROTOCOL_ACCEPTABLE }, - { "/x-bzip", NULL, NULL, NULL, NDPI_PROTOCOL_HTTP_DOWNLOAD, NDPI_PROTOCOL_CATEGORY_DOWNLOAD_FT, NDPI_PROTOCOL_ACCEPTABLE }, - { "/x-gzip", NULL, NULL, NULL, NDPI_PROTOCOL_HTTP_DOWNLOAD, NDPI_PROTOCOL_CATEGORY_DOWNLOAD_FT, NDPI_PROTOCOL_ACCEPTABLE }, - { "/x-zip", NULL, NULL, NULL, NDPI_PROTOCOL_HTTP_DOWNLOAD, NDPI_PROTOCOL_CATEGORY_DOWNLOAD_FT, NDPI_PROTOCOL_ACCEPTABLE }, - { "/zip", NULL, NULL, NULL, NDPI_PROTOCOL_HTTP_DOWNLOAD, NDPI_PROTOCOL_CATEGORY_DOWNLOAD_FT, NDPI_PROTOCOL_ACCEPTABLE }, - { "binhex", NULL, NULL, NULL, NDPI_PROTOCOL_HTTP_DOWNLOAD, NDPI_PROTOCOL_CATEGORY_DOWNLOAD_FT, NDPI_PROTOCOL_ACCEPTABLE }, - { "/base64", NULL, NULL, NULL, NDPI_PROTOCOL_HTTP_DOWNLOAD, NDPI_PROTOCOL_CATEGORY_DOWNLOAD_FT, NDPI_PROTOCOL_ACCEPTABLE }, - { "application/gnutar", NULL, NULL, NULL, NDPI_PROTOCOL_HTTP_DOWNLOAD, NDPI_PROTOCOL_CATEGORY_DOWNLOAD_FT, NDPI_PROTOCOL_ACCEPTABLE }, - { "application/x-compressed", NULL, NULL, NULL, NDPI_PROTOCOL_HTTP_DOWNLOAD, NDPI_PROTOCOL_CATEGORY_DOWNLOAD_FT, NDPI_PROTOCOL_ACCEPTABLE }, - - { NULL, NULL, NULL, 0 } -}; - /* ****************************************************** */ /* diff --git a/src/lib/ndpi_main.c b/src/lib/ndpi_main.c index b8cdf410b..b0f1383d3 100644 --- a/src/lib/ndpi_main.c +++ b/src/lib/ndpi_main.c @@ -915,12 +915,6 @@ static void init_string_based_protocols(struct ndpi_detection_module_struct *ndp // ac_automata_display(ndpi_mod->host_automa.ac_automa, 'n'); #endif - for(i=0; content_match[i].string_to_match != NULL; i++) - ndpi_add_content_subprotocol(ndpi_mod, content_match[i].string_to_match, - content_match[i].protocol_id, - content_match[i].protocol_category, - content_match[i].protocol_breed); - for(i=0; ndpi_en_bigrams[i] != NULL; i++) ndpi_string_to_automa(ndpi_mod, &ndpi_mod->bigrams_automa, (char*)ndpi_en_bigrams[i], diff --git a/src/lib/protocols/http.c b/src/lib/protocols/http.c index 4ce80f9c9..37f23e26a 100644 --- a/src/lib/protocols/http.c +++ b/src/lib/protocols/http.c @@ -65,93 +65,6 @@ static void ndpi_int_http_add_connection(struct ndpi_detection_module_struct *nd flow->http_detected = 1, flow->guessed_category = category; } -#ifdef NDPI_CONTENT_CATEGORY_FLASH -static void flash_check_http_payload(struct ndpi_detection_module_struct - *ndpi_struct, struct ndpi_flow_struct *flow) -{ - struct ndpi_packet_struct *packet = &flow->packet; - const u_int8_t *pos; - - if(packet->empty_line_position_set == 0 || (packet->empty_line_position + 10) > (packet->payload_packet_len)) - return; - - pos = &packet->payload[packet->empty_line_position] + 2; - - if(memcmp(pos, "FLV", 3) == 0 && pos[3] == 0x01 && (pos[4] == 0x01 || pos[4] == 0x04 || pos[4] == 0x05) - && pos[5] == 0x00 && pos[6] == 0x00 && pos[7] == 0x00 && pos[8] == 0x09) { - - NDPI_LOG_INFO(ndpi_struct, "found Flash content in HTTP\n"); - ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_CONTENT_CATEGORY_FLASH); - } -} -#endif - -#ifdef NDPI_CONTENT_CATEGORY_AVI -static void avi_check_http_payload(struct ndpi_detection_module_struct *ndpi_struct, struct ndpi_flow_struct *flow) -{ - struct ndpi_packet_struct *packet = &flow->packet; - - - NDPI_LOG_DBG2(ndpi_struct, "called avi_check_http_payload: %u %u %u\n", - packet->empty_line_position_set, flow->l4.tcp.http_empty_line_seen, packet->empty_line_position); - - if(packet->empty_line_position_set == 0 && flow->l4.tcp.http_empty_line_seen == 0) - return; - - if(packet->empty_line_position_set != 0 && ((packet->empty_line_position + 20) > (packet->payload_packet_len)) - && flow->l4.tcp.http_empty_line_seen == 0) { - flow->l4.tcp.http_empty_line_seen = 1; - return; - } - - if(flow->l4.tcp.http_empty_line_seen == 1) { - if(packet->payload_packet_len > 20 && memcmp(packet->payload, "RIFF", 4) == 0 - && memcmp(packet->payload + 8, "AVI LIST", 8) == 0) { - NDPI_LOG_INFO(ndpi_struct, "found Avi content in HTTP\n"); - ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_CONTENT_CATEGORY_AVI); - } - flow->l4.tcp.http_empty_line_seen = 0; - return; - } - - /** - for reference see http://msdn.microsoft.com/archive/default.asp?url=/archive/en-us/directx9_c/directx/htm/avirifffilereference.asp - **/ - if(packet->empty_line_position_set != 0) { - - u_int32_t p = packet->empty_line_position + 2; - - // check for avi header - NDPI_LOG_DBG2(ndpi_struct, "p = %u\n", p); - - if((p + 16) <= packet->payload_packet_len && memcmp(&packet->payload[p], "RIFF", 4) == 0 - && memcmp(&packet->payload[p + 8], "AVI LIST", 8) == 0) { - NDPI_LOG_INFO(ndpi_struct, "found Avi content in HTTP\n"); - ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_CONTENT_CATEGORY_AVI); - } - } -} -#endif - -static void teamviewer_check_http_payload(struct ndpi_detection_module_struct *ndpi_struct, struct ndpi_flow_struct *flow) -{ - struct ndpi_packet_struct *packet = &flow->packet; - const u_int8_t *pos; - - NDPI_LOG_DBG2(ndpi_struct, "called teamviewer_check_http_payload: %u %u %u\n", - packet->empty_line_position_set, flow->l4.tcp.http_empty_line_seen, packet->empty_line_position); - - if(packet->empty_line_position_set == 0 || (packet->empty_line_position + 5) > (packet->payload_packet_len)) - return; - - pos = &packet->payload[packet->empty_line_position] + 2; - - if(pos[0] == 0x17 && pos[1] == 0x24) { - NDPI_LOG_INFO(ndpi_struct, "found TeamViewer content in HTTP\n"); - ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_TEAMVIEWER); - } -} - static void rtsp_parse_packet_acceptline(struct ndpi_detection_module_struct *ndpi_struct, struct ndpi_flow_struct *flow) { @@ -461,15 +374,6 @@ static void check_content_type_and_change_protocol(struct ndpi_detection_module_ } } - /* search for line startin with "Icy-MetaData" */ - for (a = 0; a < packet->parsed_lines; a++) { - if(packet->line[a].len > 11 && memcmp(packet->line[a].ptr, "Icy-MetaData", 12) == 0) { - NDPI_LOG_INFO(ndpi_struct, "found MPEG: Icy-MetaData\n"); - ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_CONTENT_CATEGORY_MPEG); - return; - } - } - if(packet->content_line.ptr != NULL && packet->content_line.len != 0) { NDPI_LOG_DBG2(ndpi_struct, "Content Type line found %.*s\n", packet->content_line.len, packet->content_line.ptr); @@ -486,20 +390,8 @@ static void check_content_type_and_change_protocol(struct ndpi_detection_module_ ndpi_int_http_add_connection(ndpi_struct, flow, packet->detected_protocol_stack[0]); } -static void check_http_payload(struct ndpi_detection_module_struct *ndpi_struct, struct ndpi_flow_struct *flow) -{ - NDPI_LOG_DBG2(ndpi_struct, "called check_http_payload\n"); - -#ifdef NDPI_CONTENT_CATEGORY_FLASH - if(NDPI_COMPARE_PROTOCOL_TO_BITMASK(ndpi_struct->detection_bitmask, NDPI_CONTENT_CATEGORY_FLASH) != 0) - flash_check_http_payload(ndpi_struct, flow); -#endif -#ifdef NDPI_CONTENT_CATEGORY_AVI - if(NDPI_COMPARE_PROTOCOL_TO_BITMASK(ndpi_struct->detection_bitmask, NDPI_CONTENT_CATEGORY_AVI) != 0) - avi_check_http_payload(ndpi_struct, flow); -#endif - - teamviewer_check_http_payload(ndpi_struct, flow); +static void check_http_payload(struct ndpi_detection_module_struct *ndpi_struct, struct ndpi_flow_struct *flow) { + /* Add here your paylod code check */ } /** @@ -557,25 +449,6 @@ static u_int16_t http_request_url_offset(struct ndpi_detection_module_struct *nd static void http_bitmask_exclude_other(struct ndpi_flow_struct *flow) { -#ifdef NDPI_CONTENT_CATEGORY_MPEG - NDPI_ADD_PROTOCOL_TO_BITMASK(flow->excluded_protocol_bitmask, NDPI_CONTENT_CATEGORY_MPEG); -#endif -#ifdef NDPI_CONTENT_CATEGORY_QUICKTIME - NDPI_ADD_PROTOCOL_TO_BITMASK(flow->excluded_protocol_bitmask, NDPI_CONTENT_CATEGORY_QUICKTIME); -#endif -#ifdef NDPI_CONTENT_CATEGORY_WINDOWSMEDIA - NDPI_ADD_PROTOCOL_TO_BITMASK(flow->excluded_protocol_bitmask, NDPI_CONTENT_CATEGORY_WINDOWSMEDIA); -#endif -#ifdef NDPI_CONTENT_CATEGORY_REALMEDIA - NDPI_ADD_PROTOCOL_TO_BITMASK(flow->excluded_protocol_bitmask, NDPI_CONTENT_CATEGORY_REALMEDIA); -#endif -#ifdef NDPI_CONTENT_CATEGORY_AVI - NDPI_ADD_PROTOCOL_TO_BITMASK(flow->excluded_protocol_bitmask, NDPI_CONTENT_CATEGORY_AVI); -#endif -#ifdef NDPI_CONTENT_CATEGORY_OGG - NDPI_ADD_PROTOCOL_TO_BITMASK(flow->excluded_protocol_bitmask, NDPI_CONTENT_CATEGORY_OGG); -#endif - NDPI_ADD_PROTOCOL_TO_BITMASK(flow->excluded_protocol_bitmask, NDPI_PROTOCOL_XBOX); } @@ -985,130 +858,4 @@ void init_http_dissector(struct ndpi_detection_module_struct *ndpi_struct, u_int SAVE_DETECTION_BITMASK_AS_UNKNOWN, ADD_TO_DETECTION_BITMASK); *id += 1; - -#if 0 - ndpi_set_bitmask_protocol_detection("HTTP_Proxy", ndpi_struct, detection_bitmask, *id, - NDPI_PROTOCOL_HTTP_PROXY, - ndpi_search_http_tcp, - NDPI_SELECTION_BITMASK_PROTOCOL_V4_V6_TCP_WITH_PAYLOAD, - SAVE_DETECTION_BITMASK_AS_UNKNOWN, - ADD_TO_DETECTION_BITMASK); - *id += 1; - -#ifdef NDPI_CONTENT_CATEGORY_MPEG - ndpi_set_bitmask_protocol_detection("MPEG", ndpi_struct, detection_bitmask, *id, - NDPI_CONTENT_CATEGORY_MPEG, - ndpi_search_http_tcp, - NDPI_SELECTION_BITMASK_PROTOCOL_V4_V6_TCP_WITH_PAYLOAD, - NO_SAVE_DETECTION_BITMASK_AS_UNKNOWN, - ADD_TO_DETECTION_BITMASK); - - *id += 1; -#endif -#ifdef NDPI_CONTENT_CATEGORY_FLASH - ndpi_set_bitmask_protocol_detection("Flash", ndpi_struct, detection_bitmask, *id, - NDPI_CONTENT_CATEGORY_FLASH, - ndpi_search_http_tcp, - NDPI_SELECTION_BITMASK_PROTOCOL_V4_V6_TCP_WITH_PAYLOAD, - NO_SAVE_DETECTION_BITMASK_AS_UNKNOWN, - ADD_TO_DETECTION_BITMASK); - *id += 1; -#endif -#ifdef NDPI_CONTENT_CATEGORY_QUICKTIME - ndpi_set_bitmask_protocol_detection("QuickTime", ndpi_struct, detection_bitmask, *id, - NDPI_CONTENT_CATEGORY_QUICKTIME, - ndpi_search_http_tcp, - NDPI_SELECTION_BITMASK_PROTOCOL_V4_V6_TCP_WITH_PAYLOAD, - NO_SAVE_DETECTION_BITMASK_AS_UNKNOWN, - ADD_TO_DETECTION_BITMASK); - *id += 1; -#endif -#ifdef NDPI_CONTENT_CATEGORY_REALMEDIA - ndpi_set_bitmask_protocol_detection("RealMedia", ndpi_struct, detection_bitmask, *id, - NDPI_CONTENT_CATEGORY_REALMEDIA, - ndpi_search_http_tcp, - NDPI_SELECTION_BITMASK_PROTOCOL_V4_V6_TCP_WITH_PAYLOAD, - NO_SAVE_DETECTION_BITMASK_AS_UNKNOWN, - ADD_TO_DETECTION_BITMASK); - *id += 1; -#endif -#ifdef NDPI_CONTENT_CATEGORY_WINDOWSMEDIA - ndpi_set_bitmask_protocol_detection("WindowsMedia", ndpi_struct, detection_bitmask, *id, - NDPI_CONTENT_CATEGORY_WINDOWSMEDIA, - ndpi_search_http_tcp, - NDPI_SELECTION_BITMASK_PROTOCOL_V4_V6_TCP_WITH_PAYLOAD, - NO_SAVE_DETECTION_BITMASK_AS_UNKNOWN, - ADD_TO_DETECTION_BITMASK); - *id += 1; -#endif -#ifdef NDPI_CONTENT_CATEGORY_MMS - ndpi_set_bitmask_protocol_detection("MMS", ndpi_struct, detection_bitmask, *id, - NDPI_CONTENT_CATEGORY_MMS, - ndpi_search_http_tcp, - NDPI_SELECTION_BITMASK_PROTOCOL_V4_V6_TCP_WITH_PAYLOAD, - NO_SAVE_DETECTION_BITMASK_AS_UNKNOWN, - ADD_TO_DETECTION_BITMASK); - *id += 1; -#endif - - ndpi_set_bitmask_protocol_detection("Xbox", ndpi_struct, detection_bitmask, *id, - NDPI_PROTOCOL_XBOX, - ndpi_search_http_tcp, - NDPI_SELECTION_BITMASK_PROTOCOL_V4_V6_TCP_WITH_PAYLOAD, - NO_SAVE_DETECTION_BITMASK_AS_UNKNOWN, - ADD_TO_DETECTION_BITMASK); - *id += 1; - - ndpi_set_bitmask_protocol_detection("QQ", ndpi_struct, detection_bitmask, *id, - NDPI_PROTOCOL_QQ, - ndpi_search_http_tcp, - NDPI_SELECTION_BITMASK_PROTOCOL_V4_V6_TCP_WITH_PAYLOAD, - NO_SAVE_DETECTION_BITMASK_AS_UNKNOWN, - ADD_TO_DETECTION_BITMASK); - *id += 1; - -#ifdef NDPI_CONTENT_CATEGORY_AVI - ndpi_set_bitmask_protocol_detection("AVI", ndpi_struct, detection_bitmask, *id, - NDPI_CONTENT_CATEGORY_AVI, - ndpi_search_http_tcp, - NDPI_SELECTION_BITMASK_PROTOCOL_V4_V6_TCP_WITH_PAYLOAD, - NO_SAVE_DETECTION_BITMASK_AS_UNKNOWN, - ADD_TO_DETECTION_BITMASK); - *id += 1; -#endif -#ifdef NDPI_CONTENT_CATEGORY_OGG - ndpi_set_bitmask_protocol_detection("OggVorbis", ndpi_struct, detection_bitmask, *id, - NDPI_CONTENT_CATEGORY_OGG, - ndpi_search_http_tcp, - NDPI_SELECTION_BITMASK_PROTOCOL_V4_V6_TCP_WITH_PAYLOAD, - NO_SAVE_DETECTION_BITMASK_AS_UNKNOWN, - ADD_TO_DETECTION_BITMASK); - *id += 1; -#endif - - /* Update excluded protocol bitmask */ - NDPI_BITMASK_SET(ndpi_struct->callback_buffer[a].excluded_protocol_bitmask, - ndpi_struct->callback_buffer[a].detection_bitmask); - - /*Delete protocol from excluded protocol bitmask*/ - NDPI_DEL_PROTOCOL_FROM_BITMASK(ndpi_struct->callback_buffer[a].excluded_protocol_bitmask, NDPI_PROTOCOL_UNKNOWN); - - NDPI_DEL_PROTOCOL_FROM_BITMASK(ndpi_struct->callback_buffer[a].excluded_protocol_bitmask, NDPI_PROTOCOL_QQ); - -#ifdef NDPI_CONTENT_CATEGORY_FLASH - NDPI_DEL_PROTOCOL_FROM_BITMASK(ndpi_struct->callback_buffer[a].excluded_protocol_bitmask, NDPI_CONTENT_CATEGORY_FLASH); -#endif - - NDPI_DEL_PROTOCOL_FROM_BITMASK(ndpi_struct->callback_buffer[a].excluded_protocol_bitmask, NDPI_CONTENT_CATEGORY_MMS); - - NDPI_DEL_PROTOCOL_FROM_BITMASK(ndpi_struct->callback_buffer[a].excluded_protocol_bitmask, NDPI_PROTOCOL_XBOX); - - NDPI_BITMASK_SET(ndpi_struct->generic_http_packet_bitmask, ndpi_struct->callback_buffer[a].detection_bitmask); - - NDPI_DEL_PROTOCOL_FROM_BITMASK(ndpi_struct->generic_http_packet_bitmask, NDPI_PROTOCOL_UNKNOWN); - - /* Update callback_buffer index */ - a++; - -#endif } -- cgit v1.2.3 From 6c1d236a03f6c41f14d0d3c8a7627548fcd55779 Mon Sep 17 00:00:00 2001 From: Luca Deri Date: Sun, 10 Mar 2019 21:44:11 +0100 Subject: Category cleanup --- src/lib/ndpi_main.c | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) (limited to 'src/lib') diff --git a/src/lib/ndpi_main.c b/src/lib/ndpi_main.c index b0f1383d3..2471a0eb1 100644 --- a/src/lib/ndpi_main.c +++ b/src/lib/ndpi_main.c @@ -5698,14 +5698,6 @@ static const char* categories[] = { "", "", "", - "AVI", - "Flash", - "OGG", - "MPEG", - "QuickTime", - "RealMedia", - "WindowsMedia", - "Webm", /* 32 */ "Music", "Video", "Shopping", @@ -5791,9 +5783,7 @@ const char* ndpi_category_get_name(struct ndpi_detection_module_struct *ndpi_mod if((!ndpi_mod) || (category >= NDPI_PROTOCOL_NUM_CATEGORIES)) return(NULL); - if((category < NDPI_PROTOCOL_CATEGORY_CUSTOM_1) || (category >= CUSTOM_CATEGORY_MINING)) - return(categories[category]); - else { + if((category >= NDPI_PROTOCOL_CATEGORY_CUSTOM_1) && (category <= NDPI_PROTOCOL_CATEGORY_CUSTOM_5)) { switch(category) { case NDPI_PROTOCOL_CATEGORY_CUSTOM_1: return(ndpi_mod->custom_category_labels[0]); @@ -5810,7 +5800,8 @@ const char* ndpi_category_get_name(struct ndpi_detection_module_struct *ndpi_mod default: return("Unspecified"); } - } + } else + return(categories[category]); } /* ****************************************************** */ -- cgit v1.2.3 From b1bd7d6f8c7a18beabaf8224056f13c2193dce03 Mon Sep 17 00:00:00 2001 From: chiehminw Date: Tue, 12 Mar 2019 16:11:33 +0800 Subject: Add NDPI prefix for HTTP_METHOD enum to avoid name collisions ref: https://stackoverflow.com/questions/35380279/avoid-name-collisions-with-enum-in-c-c99 Signed-off-by: chiehminw --- src/include/ndpi_typedefs.h | 18 +++++++++--------- src/lib/protocols/http.c | 22 +++++++++++----------- 2 files changed, 20 insertions(+), 20 deletions(-) (limited to 'src/lib') diff --git a/src/include/ndpi_typedefs.h b/src/include/ndpi_typedefs.h index a5f1864e0..e561deb47 100644 --- a/src/include/ndpi_typedefs.h +++ b/src/include/ndpi_typedefs.h @@ -356,15 +356,15 @@ PACK_ON struct tinc_cache_entry { } PACK_OFF; typedef enum { - HTTP_METHOD_UNKNOWN = 0, - HTTP_METHOD_OPTIONS, - HTTP_METHOD_GET, - HTTP_METHOD_HEAD, - HTTP_METHOD_POST, - HTTP_METHOD_PUT, - HTTP_METHOD_DELETE, - HTTP_METHOD_TRACE, - HTTP_METHOD_CONNECT + NDPI_HTTP_METHOD_UNKNOWN = 0, + NDPI_HTTP_METHOD_OPTIONS, + NDPI_HTTP_METHOD_GET, + NDPI_HTTP_METHOD_HEAD, + NDPI_HTTP_METHOD_POST, + NDPI_HTTP_METHOD_PUT, + NDPI_HTTP_METHOD_DELETE, + NDPI_HTTP_METHOD_TRACE, + NDPI_HTTP_METHOD_CONNECT } ndpi_http_method; struct ndpi_lru_cache { diff --git a/src/lib/protocols/http.c b/src/lib/protocols/http.c index 4ce80f9c9..661e43bf2 100644 --- a/src/lib/protocols/http.c +++ b/src/lib/protocols/http.c @@ -260,25 +260,25 @@ static void check_content_type_and_change_protocol(struct ndpi_detection_module_ } if(flow->packet.http_method.len < 3) - flow->http.method = HTTP_METHOD_UNKNOWN; + flow->http.method = NDPI_HTTP_METHOD_UNKNOWN; else { switch(flow->packet.http_method.ptr[0]) { - case 'O': flow->http.method = HTTP_METHOD_OPTIONS; break; - case 'G': flow->http.method = HTTP_METHOD_GET; break; - case 'H': flow->http.method = HTTP_METHOD_HEAD; break; + case 'O': flow->http.method = NDPI_HTTP_METHOD_OPTIONS; break; + case 'G': flow->http.method = NDPI_HTTP_METHOD_GET; break; + case 'H': flow->http.method = NDPI_HTTP_METHOD_HEAD; break; case 'P': switch(flow->packet.http_method.ptr[1]) { - case 'O': flow->http.method = HTTP_METHOD_POST; break; - case 'U': flow->http.method = HTTP_METHOD_PUT; break; + case 'O': flow->http.method = NDPI_HTTP_METHOD_POST; break; + case 'U': flow->http.method = NDPI_HTTP_METHOD_PUT; break; } break; - case 'D': flow->http.method = HTTP_METHOD_DELETE; break; - case 'T': flow->http.method = HTTP_METHOD_TRACE; break; - case 'C': flow->http.method = HTTP_METHOD_CONNECT; break; + case 'D': flow->http.method = NDPI_HTTP_METHOD_DELETE; break; + case 'T': flow->http.method = NDPI_HTTP_METHOD_TRACE; break; + case 'C': flow->http.method = NDPI_HTTP_METHOD_CONNECT; break; default: - flow->http.method = HTTP_METHOD_UNKNOWN; + flow->http.method = NDPI_HTTP_METHOD_UNKNOWN; break; } } @@ -949,7 +949,7 @@ void ndpi_search_http_tcp(struct ndpi_detection_module_struct *ndpi_struct, ndpi_http_method ndpi_get_http_method(struct ndpi_detection_module_struct *ndpi_mod, struct ndpi_flow_struct *flow) { if(!flow) - return(HTTP_METHOD_UNKNOWN); + return(NDPI_HTTP_METHOD_UNKNOWN); else return(flow->http.method); } -- cgit v1.2.3 From 22e431ca67ee27c623309cf073e277b0f65a8272 Mon Sep 17 00:00:00 2001 From: Luca Date: Tue, 12 Mar 2019 18:01:14 +0100 Subject: Sporify fix --- src/lib/ndpi_main.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/lib') diff --git a/src/lib/ndpi_main.c b/src/lib/ndpi_main.c index 2471a0eb1..b8bfc77d5 100644 --- a/src/lib/ndpi_main.c +++ b/src/lib/ndpi_main.c @@ -1721,7 +1721,7 @@ static void ndpi_init_protocol_defaults(struct ndpi_detection_module_struct *ndp ndpi_build_default_ports(ports_b, 17500, 0, 0, 0, 0) /* UDP */); ndpi_set_proto_defaults(ndpi_mod, NDPI_PROTOCOL_ACCEPTABLE, NDPI_PROTOCOL_SPOTIFY, 0 /* can_have_a_subprotocol */, no_master, - no_master, "Spotify", NDPI_PROTOCOL_CATEGORY_STREAMING, + no_master, "Spotify", NDPI_PROTOCOL_CATEGORY_MUSIC, ndpi_build_default_ports(ports_a, 0, 0, 0, 0, 0) /* TCP */, ndpi_build_default_ports(ports_b, 0, 0, 0, 0, 0) /* UDP */); ndpi_set_proto_defaults(ndpi_mod, NDPI_PROTOCOL_ACCEPTABLE, NDPI_PROTOCOL_LISP, -- cgit v1.2.3 From b1282da5424cfc7fbe331ac5256148780a5fa2fb Mon Sep 17 00:00:00 2001 From: Guido Falsi Date: Tue, 12 Mar 2019 18:12:18 +0100 Subject: The le32toh() function used in some places on BSD OSes require sys/endian.h to be included. --- src/lib/ndpi_main.c | 4 ++++ src/lib/protocols/quic.c | 4 ++++ 2 files changed, 8 insertions(+) (limited to 'src/lib') diff --git a/src/lib/ndpi_main.c b/src/lib/ndpi_main.c index 2471a0eb1..4e9f37248 100644 --- a/src/lib/ndpi_main.c +++ b/src/lib/ndpi_main.c @@ -41,6 +41,10 @@ #include #endif +#if defined __FreeBSD__ || defined __NetBSD__ || defined __OpenBSD__ +#include +#endif + #include "ndpi_content_match.c.inc" #include "third_party/include/ndpi_patricia.h" #include "third_party/include/ht_hash.h" diff --git a/src/lib/protocols/quic.c b/src/lib/protocols/quic.c index 322eb9be7..d14538e0d 100644 --- a/src/lib/protocols/quic.c +++ b/src/lib/protocols/quic.c @@ -22,6 +22,10 @@ * */ +#if defined __FreeBSD__ || defined __NetBSD__ || defined __OpenBSD__ +#include +#endif + #include "ndpi_protocol_ids.h" #define NDPI_CURRENT_PROTO NDPI_PROTOCOL_QUIC -- cgit v1.2.3 From 43890b31fc45a76ac10e49a747ec925f77102bb7 Mon Sep 17 00:00:00 2001 From: chiehminw Date: Wed, 13 Mar 2019 15:04:10 +0800 Subject: Use relative path for library link relative link path can preserve link relation after packaging and doplying to new machine ex: $ make DESTDIR=/tmp/ndpi install $ tar czf ndpi.tar.gz /tmp/ndpi $ tar xf ndpi.tar.gz -C /lib Orignal install script will caused the /lib/libndpi.so.2 links to /tmp/ndpi/libndpi.so.2.9.0 which is not intended Signed-off-by: chiehminw --- src/lib/Makefile.in | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src/lib') diff --git a/src/lib/Makefile.in b/src/lib/Makefile.in index 8b0853dd4..cadd65ddd 100644 --- a/src/lib/Makefile.in +++ b/src/lib/Makefile.in @@ -61,7 +61,7 @@ distclean: clean install: $(NDPI_LIBS) mkdir -p $(DESTDIR)$(libdir) cp $(NDPI_LIBS) $(DESTDIR)$(libdir)/ - ln -Ffs $(DESTDIR)$(libdir)/$(NDPI_LIB_SHARED) $(DESTDIR)$(libdir)/$(NDPI_LIB_SHARED_BASE) - ln -Ffs $(DESTDIR)$(libdir)/$(NDPI_LIB_SHARED) $(DESTDIR)$(libdir)/$(NDPI_LIB_SHARED_BASE).$(NDPI_VERSION_MAJOR) + cp -P $(NDPI_LIB_SHARED_BASE) $(DESTDIR)$(libdir)/ + cp -P $(NDPI_LIB_SHARED_BASE).$(NDPI_VERSION_MAJOR) $(DESTDIR)$(libdir)/ mkdir -p $(DESTDIR)$(includedir) cp ../include/*.h $(DESTDIR)$(includedir) -- cgit v1.2.3 From 54f90c7556ca2ec3ee636000d8e59328d94101bc Mon Sep 17 00:00:00 2001 From: Luca Deri Date: Thu, 14 Mar 2019 21:50:09 +0100 Subject: Added fix to avoid FTP false positives --- src/lib/protocols/ftp_data.c | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) (limited to 'src/lib') diff --git a/src/lib/protocols/ftp_data.c b/src/lib/protocols/ftp_data.c index 8d3e6fa8c..a6b0d2f38 100644 --- a/src/lib/protocols/ftp_data.c +++ b/src/lib/protocols/ftp_data.c @@ -220,16 +220,24 @@ static int ndpi_match_file_header(struct ndpi_detection_module_struct *ndpi_stru static void ndpi_check_ftp_data(struct ndpi_detection_module_struct *ndpi_struct, struct ndpi_flow_struct *flow) { struct ndpi_packet_struct *packet = &flow->packet; - if((packet->payload_packet_len > 0) - && (ndpi_match_file_header(ndpi_struct, flow) - || ndpi_match_ftp_data_directory(ndpi_struct, flow) - || ndpi_match_ftp_data_port(ndpi_struct, flow) - ) - ) { - NDPI_LOG_INFO(ndpi_struct, "found FTP_DATA request\n"); - ndpi_int_ftp_data_add_connection(ndpi_struct, flow); - } else - NDPI_EXCLUDE_PROTO(ndpi_struct, flow); + /* + Make sure we see the beginning of the connection as otherwise we might have + false positive results + */ + if(flow->l4.tcp.seen_syn) { + if((packet->payload_packet_len > 0) + && (ndpi_match_file_header(ndpi_struct, flow) + || ndpi_match_ftp_data_directory(ndpi_struct, flow) + || ndpi_match_ftp_data_port(ndpi_struct, flow) + ) + ) { + NDPI_LOG_INFO(ndpi_struct, "found FTP_DATA request\n"); + ndpi_int_ftp_data_add_connection(ndpi_struct, flow); + return; + } + } + + NDPI_EXCLUDE_PROTO(ndpi_struct, flow); } void ndpi_search_ftp_data(struct ndpi_detection_module_struct *ndpi_struct, struct ndpi_flow_struct *flow) { -- cgit v1.2.3 From 7237f7081f2b89a53aaf306d1b881fba09a91407 Mon Sep 17 00:00:00 2001 From: Luca Deri Date: Thu, 14 Mar 2019 22:07:21 +0100 Subject: Improved FTP dissection --- src/lib/protocols/ftp_data.c | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) (limited to 'src/lib') diff --git a/src/lib/protocols/ftp_data.c b/src/lib/protocols/ftp_data.c index a6b0d2f38..7c646c363 100644 --- a/src/lib/protocols/ftp_data.c +++ b/src/lib/protocols/ftp_data.c @@ -49,16 +49,22 @@ static int ndpi_match_ftp_data_directory(struct ndpi_detection_module_struct *nd struct ndpi_packet_struct *packet = &flow->packet; u_int32_t payload_len = packet->payload_packet_len; - if((payload_len >= 4) - && ((packet->payload[0] == '-') || (packet->payload[0] == 'd')) - && ((packet->payload[1] == '-') || (packet->payload[1] == 'r')) - && ((packet->payload[2] == '-') || (packet->payload[2] == 'w')) - && ((packet->payload[3] == '-') || (packet->payload[3] == 'x'))) { + if(payload_len > 10) { + int i; - return 1; + if(!((packet->payload[0] == '-') || (packet->payload[0] == 'd'))) + return(0); + + for(i=0; i<9; i += 3) + if(((packet->payload[1+i] == '-') || (packet->payload[1+i] == 'r')) + && ((packet->payload[2+i] == '-') || (packet->payload[2+i] == 'w')) + && ((packet->payload[3+i] == '-') || (packet->payload[3+i] == 'x'))) { + ; + } else + return 0; } - return 0; + return 1; } static int ndpi_match_file_header(struct ndpi_detection_module_struct *ndpi_struct, struct ndpi_flow_struct *flow) { -- cgit v1.2.3 From 6c9fbc2ddb9bbdc054580ff6dd4e7f5b7ea8a452 Mon Sep 17 00:00:00 2001 From: emanuele-f Date: Mon, 18 Mar 2019 13:29:08 +0100 Subject: Fix invalid categories labels array length and add check --- src/lib/ndpi_main.c | 223 +++++++++++++++++++++++++++------------------------- 1 file changed, 116 insertions(+), 107 deletions(-) (limited to 'src/lib') diff --git a/src/lib/ndpi_main.c b/src/lib/ndpi_main.c index b8bfc77d5..a99d964ec 100644 --- a/src/lib/ndpi_main.c +++ b/src/lib/ndpi_main.c @@ -314,6 +314,116 @@ int strncasecmp(s1, s2, n) /* ****************************************** */ +/* Keep it in order and in sync with ndpi_protocol_category_t in ndpi_typedefs.h */ +static const char* categories[] = { + "Unspecified", + "Media", + "VPN", + "Email", + "DataTransfer", + "Web", + "SocialNetwork", + "Download-FileTransfer-FileSharing", + "Game", + "Chat", + "VoIP", + "Database", + "RemoteAccess", + "Cloud", + "Network", + "Collaborative", + "RPC", + "Streaming", + "System", + "SoftwareUpdate", + "", + "", + "", + "", + "", + "Music", + "Video", + "Shopping", + "Productivity", + "FileSharing", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "Mining", /* 99 */ + "Malware", + "Advertisement", + "Banned_Site", + "Site_Unavailable" +}; + +/* ****************************************** */ + /* Forward */ static void addDefaultPort(struct ndpi_detection_module_struct *ndpi_mod, ndpi_port_range *range, @@ -2239,6 +2349,12 @@ struct ndpi_detection_module_struct *ndpi_init_detection_module(void) { ndpi_str->bigrams_automa.ac_automa = ac_automata_init(ac_match_handler); ndpi_str->impossible_bigrams_automa.ac_automa = ac_automata_init(ac_match_handler); + if((sizeof(categories)/sizeof(char*)) != NDPI_PROTOCOL_NUM_CATEGORIES) { + NDPI_LOG_ERR(ndpi_str, "[NDPI] invalid categories length: expected %u, got %u\n", + NDPI_PROTOCOL_NUM_CATEGORIES, (sizeof(categories)/sizeof(char*))); + return(NULL); + } + #ifdef HAVE_HYPERSCAN ndpi_str->custom_categories.num_to_load = 0, ndpi_str->custom_categories.to_load = NULL; ndpi_str->custom_categories.hostnames = NULL; @@ -5671,113 +5787,6 @@ void ndpi_category_set_name(struct ndpi_detection_module_struct *ndpi_mod, /* ****************************************************** */ -/* Keep it in order and in sync with ndpi_protocol_category_t in ndpi_typedefs.h */ -static const char* categories[] = { - "Unspecified", - "Media", - "VPN", - "Email", - "DataTransfer", - "Web", - "SocialNetwork", - "Download-FileTransfer-FileSharing", - "Game", - "Chat", - "VoIP", - "Database", - "RemoteAccess", - "Cloud", - "Network", - "Collaborative", - "RPC", - "Streaming", - "System", - "SoftwareUpdate", - "", - "", - "", - "", - "", - "Music", - "Video", - "Shopping", - "Productivity", - "FileSharing", - "", - "", - "", - - "", - "", - "", - "", - "", - "", - "", - "", - "", - "", - - "", - "", - "", - "", - "", - "", - "", - "", - "", - "", - - "", - "", - "", - "", - "", - "", - "", - "", - "", - "", - - "", - "", - "", - "", - "", - "", - "", - "", - "", - "", - - "", - "", - "", - "", - "", - "", - "", - "", - "", - "", - - "", - "", - "", - "", - "", - "", - "", - "", - - "Mining", /* 99 */ - "Malware", - "Advertisement", - "Banned_Site", - "Site_Unavailable" -}; - const char* ndpi_category_get_name(struct ndpi_detection_module_struct *ndpi_mod, ndpi_protocol_category_t category) { if((!ndpi_mod) || (category >= NDPI_PROTOCOL_NUM_CATEGORIES)) -- cgit v1.2.3 From db91837647737b3d57359dd6d7bc8474644f8043 Mon Sep 17 00:00:00 2001 From: chiehminw Date: Wed, 20 Mar 2019 17:22:37 +0800 Subject: retreive ssl certificate origanization Signed-off-by: chiehminw --- src/include/ndpi_typedefs.h | 2 +- src/lib/protocols/ssl.c | 69 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 70 insertions(+), 1 deletion(-) (limited to 'src/lib') diff --git a/src/include/ndpi_typedefs.h b/src/include/ndpi_typedefs.h index 39995d662..0c71f9de4 100644 --- a/src/include/ndpi_typedefs.h +++ b/src/include/ndpi_typedefs.h @@ -1085,7 +1085,7 @@ struct ndpi_flow_struct { struct { struct { - char client_certificate[64], server_certificate[64]; + char client_certificate[64], server_certificate[64], server_organization[64]; } ssl; struct { diff --git a/src/lib/protocols/ssl.c b/src/lib/protocols/ssl.c index ff6b47a0f..845f20de6 100644 --- a/src/lib/protocols/ssl.c +++ b/src/lib/protocols/ssl.c @@ -335,18 +335,87 @@ int getSSLcertificate(struct ndpi_detection_module_struct *ndpi_struct, return(0); /* Not found */ } +void getSSLorganization(struct ndpi_detection_module_struct *ndpi_struct, + struct ndpi_flow_struct *flow, + char *buffer, int buffer_len) { + struct ndpi_packet_struct *packet = &flow->packet; + + if(packet->payload[0] != 0x16 /* Handshake */) + return; + + u_int16_t total_len = (packet->payload[3] << 8) + packet->payload[4] + 5 /* SSL Header */; + u_int8_t handshake_protocol = packet->payload[5]; /* handshake protocol a bit misleading, it is message type according TLS specs */ + + if(handshake_protocol != 0x02 && handshake_protocol != 0xb /* Server Hello and Certificate message types are interesting for us */) + return; + + /* Truncate total len, search at least in incomplete packet */ + if(total_len > packet->payload_packet_len) + total_len = packet->payload_packet_len; + + memset(buffer, 0, buffer_len); + + /* Check after handshake protocol header (5 bytes) and message header (4 bytes) */ + u_int num_found = 0; + u_int i, j; + for(i = 9; i < packet->payload_packet_len-4; i++) { + /* Organization OID: 2.5.4.10 */ + if((packet->payload[i] == 0x55) && (packet->payload[i+1] == 0x04) && (packet->payload[i+2] == 0x0a)) { + u_int8_t type_tag = packet->payload[i+3]; // 0x0c: utf8string / 0x13: printable_string + u_int8_t server_len = packet->payload[i+4]; + + num_found++; + /* what we want is subject certificate, so we bypass the issuer certificate */ + if (num_found != 2) continue; + + // packet is truncated... further inspection is not needed + if(i+4+server_len >= packet->payload_packet_len) { + break; + } + + char *server_org = (char*)&packet->payload[i+5]; + + u_int len = (u_int)ndpi_min(server_len, buffer_len-1); + strncpy(buffer, server_org, len); + buffer[len] = '\0'; + + // check if organization string are all printable + u_int8_t is_printable = 1; + for (j = 0; j < len; j++) { + if(!ndpi_isprint(buffer[j])) { + is_printable = 0; + break; + } + } + + if (is_printable == 1) { + snprintf(flow->protos.stun_ssl.ssl.server_organization, + sizeof(flow->protos.stun_ssl.ssl.server_organization), "%s", buffer); +#ifdef CERTIFICATE_DEBUG + printf("Certificate origanization: %s\n", flow->protos.stun_ssl.ssl.server_organization); +#endif + } + } + } +} + int sslTryAndRetrieveServerCertificate(struct ndpi_detection_module_struct *ndpi_struct, struct ndpi_flow_struct *flow) { struct ndpi_packet_struct *packet = &flow->packet; /* consider only specific SSL packets (handshake) */ if((packet->payload_packet_len > 9) && (packet->payload[0] == 0x16)) { char certificate[64]; + char organization[64]; int rc; certificate[0] = '\0'; rc = getSSLcertificate(ndpi_struct, flow, certificate, sizeof(certificate)); packet->ssl_certificate_num_checks++; if (rc > 0) { + // try fetch server organization once server certificate is found + organization[0] = '\0'; + getSSLorganization(ndpi_struct, flow, organization, sizeof(organization)); + packet->ssl_certificate_detected++; if ((flow->l4.tcp.ssl_seen_server_cert == 1) && (flow->protos.stun_ssl.ssl.server_certificate[0] != '\0')) /* 0 means we're done processing extra packets (since we found what we wanted) */ -- cgit v1.2.3 From 96ed31874dd62caef9e6f4dfe5c3825e88151c20 Mon Sep 17 00:00:00 2001 From: chiehminw Date: Wed, 20 Mar 2019 17:25:39 +0800 Subject: we only need one dot to confirm it is a valid domain name github.com is valid but can not be retreive by server certificate parsing Signed-off-by: chiehminw --- src/lib/protocols/ssl.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src/lib') diff --git a/src/lib/protocols/ssl.c b/src/lib/protocols/ssl.c index 845f20de6..e6cc48bba 100644 --- a/src/lib/protocols/ssl.c +++ b/src/lib/protocols/ssl.c @@ -216,11 +216,11 @@ int getSSLcertificate(struct ndpi_detection_module_struct *ndpi_struct, break; } else if(buffer[j] == '.') { num_dots++; - if(num_dots >=2) break; + if(num_dots >=1) break; } } - if(num_dots >= 2) { + if(num_dots >= 1) { if(!ndpi_struct->disable_metadata_export) { stripCertificateTrailer(buffer, buffer_len); snprintf(flow->protos.stun_ssl.ssl.server_certificate, -- cgit v1.2.3 From 5656a41f696e8073de28be5bdbd909fda108ee78 Mon Sep 17 00:00:00 2001 From: Luca Date: Fri, 22 Mar 2019 12:36:42 +0100 Subject: Improved snapchat detection --- src/lib/ndpi_content_match.c.inc | 4 ++++ src/lib/ndpi_main.c | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) (limited to 'src/lib') diff --git a/src/lib/ndpi_content_match.c.inc b/src/lib/ndpi_content_match.c.inc index 3261618b9..3a8a9664f 100644 --- a/src/lib/ndpi_content_match.c.inc +++ b/src/lib/ndpi_content_match.c.inc @@ -8246,6 +8246,10 @@ ndpi_protocol_match host_match[] = { { "feelinsonice.com", NULL, "\\.feelsonice" TLD, "Snapchat", NDPI_PROTOCOL_SNAPCHAT, NDPI_PROTOCOL_CATEGORY_SOCIAL_NETWORK, NDPI_PROTOCOL_FUN }, { ".snapchat.", NULL, "\\.snapchat" TLD, "Snapchat", NDPI_PROTOCOL_SNAPCHAT, NDPI_PROTOCOL_CATEGORY_SOCIAL_NETWORK, NDPI_PROTOCOL_FUN }, { ".snapads.", NULL, "\\.snapads" TLD, "Snapchat", NDPI_PROTOCOL_SNAPCHAT, NDPI_PROTOCOL_CATEGORY_SOCIAL_NETWORK, NDPI_PROTOCOL_FUN }, + { ".sc-cdn.net", NULL, "\\.sc-cdn\\.net" TLD, "Snapchat", NDPI_PROTOCOL_SNAPCHAT, NDPI_PROTOCOL_CATEGORY_SOCIAL_NETWORK, NDPI_PROTOCOL_FUN }, + { ".sc-prod.net", NULL, "\\.sc-prod\\.net" TLD, "Snapchat", NDPI_PROTOCOL_SNAPCHAT, NDPI_PROTOCOL_CATEGORY_SOCIAL_NETWORK, NDPI_PROTOCOL_FUN }, + { ".sc-jpl.com", NULL, "\\.sc-jpl\\.com" TLD, "Snapchat", NDPI_PROTOCOL_SNAPCHAT, NDPI_PROTOCOL_CATEGORY_SOCIAL_NETWORK, NDPI_PROTOCOL_FUN }, + { "sc-analytics.appspot.com", NULL, "sc-analytics\\.appspot\\.com", "Snapchat", NDPI_PROTOCOL_SNAPCHAT, NDPI_PROTOCOL_CATEGORY_SOCIAL_NETWORK, NDPI_PROTOCOL_FUN }, { ".waze.com", NULL, "\\.waze" TLD, "Waze", NDPI_PROTOCOL_WAZE, NDPI_PROTOCOL_CATEGORY_WEB, NDPI_PROTOCOL_ACCEPTABLE }, diff --git a/src/lib/ndpi_main.c b/src/lib/ndpi_main.c index a99d964ec..ae56fa5cc 100644 --- a/src/lib/ndpi_main.c +++ b/src/lib/ndpi_main.c @@ -2351,7 +2351,7 @@ struct ndpi_detection_module_struct *ndpi_init_detection_module(void) { if((sizeof(categories)/sizeof(char*)) != NDPI_PROTOCOL_NUM_CATEGORIES) { NDPI_LOG_ERR(ndpi_str, "[NDPI] invalid categories length: expected %u, got %u\n", - NDPI_PROTOCOL_NUM_CATEGORIES, (sizeof(categories)/sizeof(char*))); + NDPI_PROTOCOL_NUM_CATEGORIES, (unsigned int)(sizeof(categories)/sizeof(char*))); return(NULL); } -- cgit v1.2.3 From f47be6ef6045a97a20f7a929d15a0354260c0414 Mon Sep 17 00:00:00 2001 From: Luca Deri Date: Sat, 23 Mar 2019 10:32:18 +0100 Subject: Added SSL organization support --- example/ndpiReader.c | 1 + example/ndpi_util.c | 2 ++ example/ndpi_util.h | 2 +- src/lib/ndpi_main.c | 2 +- 4 files changed, 5 insertions(+), 2 deletions(-) (limited to 'src/lib') diff --git a/example/ndpiReader.c b/example/ndpiReader.c index fe0ea879b..4244e3745 100644 --- a/example/ndpiReader.c +++ b/example/ndpiReader.c @@ -804,6 +804,7 @@ static void printFlow(u_int16_t id, struct ndpi_flow_info *flow, u_int16_t threa if(flow->ssh_ssl.client_info[0] != '\0') fprintf(out, "[client: %s]", flow->ssh_ssl.client_info); if(flow->ssh_ssl.server_info[0] != '\0') fprintf(out, "[server: %s]", flow->ssh_ssl.server_info); + if(flow->ssh_ssl.server_organization[0] != '\0') fprintf(out, "[organization: %s]", flow->ssh_ssl.server_organization); if(flow->bittorent_hash[0] != '\0') fprintf(out, "[BT Hash: %s]", flow->bittorent_hash); fprintf(out, "\n"); diff --git a/example/ndpi_util.c b/example/ndpi_util.c index 3a2a66c97..9e1e72132 100644 --- a/example/ndpi_util.c +++ b/example/ndpi_util.c @@ -566,6 +566,8 @@ void process_ndpi_collected_info(struct ndpi_workflow * workflow, struct ndpi_fl flow->ndpi_flow->protos.stun_ssl.ssl.client_certificate); snprintf(flow->ssh_ssl.server_info, sizeof(flow->ssh_ssl.server_info), "%s", flow->ndpi_flow->protos.stun_ssl.ssl.server_certificate); + snprintf(flow->ssh_ssl.server_organization, sizeof(flow->ssh_ssl.server_organization), "%s", + flow->ndpi_flow->protos.stun_ssl.ssl.server_organization); } } diff --git a/example/ndpi_util.h b/example/ndpi_util.h index eb9ab8e65..0a5a3b8c2 100644 --- a/example/ndpi_util.h +++ b/example/ndpi_util.h @@ -97,7 +97,7 @@ typedef struct ndpi_flow_info { char bittorent_hash[41]; struct { - char client_info[64], server_info[64]; + char client_info[64], server_info[64], server_organization[64]; } ssh_ssl; void *src_id, *dst_id; diff --git a/src/lib/ndpi_main.c b/src/lib/ndpi_main.c index 047167ac3..59de7a763 100644 --- a/src/lib/ndpi_main.c +++ b/src/lib/ndpi_main.c @@ -1,7 +1,7 @@ /* * ndpi_main.c * - * Copyright (C) 2011-18 - ntop.org + * Copyright (C) 2011-19 - ntop.org * * This file is part of nDPI, an open source deep packet inspection * library based on the OpenDPI and PACE technology by ipoque GmbH -- cgit v1.2.3 From 0b5e5a5f82ce87c14a5ecd588ed1dd0b08ff5e18 Mon Sep 17 00:00:00 2001 From: Luca Deri Date: Mon, 25 Mar 2019 22:08:39 +0100 Subject: SSL cleanup --- src/include/ndpi_typedefs.h | 5 ++- src/lib/protocols/ssl.c | 89 +++++++++++++++++++++++++-------------------- 2 files changed, 53 insertions(+), 41 deletions(-) (limited to 'src/lib') diff --git a/src/include/ndpi_typedefs.h b/src/include/ndpi_typedefs.h index c49768f64..92c011b3d 100644 --- a/src/include/ndpi_typedefs.h +++ b/src/include/ndpi_typedefs.h @@ -541,7 +541,10 @@ struct ndpi_flow_tcp_struct { u_int32_t telnet_stage:2; // 0 - 2 /* NDPI_PROTOCOL_SSL */ - u_int8_t ssl_seen_client_cert:1, ssl_seen_server_cert:1, ssl_stage:2; // 0 - 5 + u_int8_t ssl_seen_client_cert:1, + ssl_seen_server_cert:1, + ssl_seen_certificate:1, + ssl_stage:2; // 0 - 5 /* NDPI_PROTOCOL_POSTGRES */ u_int32_t postgres_stage:3; diff --git a/src/lib/protocols/ssl.c b/src/lib/protocols/ssl.c index e6cc48bba..05988a8d4 100644 --- a/src/lib/protocols/ssl.c +++ b/src/lib/protocols/ssl.c @@ -27,7 +27,7 @@ #include "ndpi_api.h" -//#define CERTIFICATE_DEBUG 1 +// #define CERTIFICATE_DEBUG 1 #define NDPI_MAX_SSL_REQUEST_SIZE 10000 @@ -152,8 +152,9 @@ int getSSLcertificate(struct ndpi_detection_module_struct *ndpi_struct, #ifdef CERTIFICATE_DEBUG { u_int16_t ssl_version = (packet->payload[1] << 8) + packet->payload[2]; - - printf("SSL [version: %u]\n", ssl_version); + u_int16_t ssl_len = (packet->payload[3] << 8) + packet->payload[4]; + + printf("SSL Record [version: 0x%02X][len: %u]\n", ssl_version, ssl_len); } #endif @@ -175,10 +176,18 @@ int getSSLcertificate(struct ndpi_detection_module_struct *ndpi_struct, if(total_len > 4) { int i; - if(handshake_protocol == 0x02 || handshake_protocol == 0xb /* Server Hello and Certificate message types are interesting for us */) { +#ifdef CERTIFICATE_DEBUG + printf("SSL [len: %u][handshake_protocol: %02X]\n", packet->payload_packet_len, handshake_protocol); +#endif + + if((handshake_protocol == 0x02) + || (handshake_protocol == 0xb) /* Server Hello and Certificate message types are interesting for us */) { u_int num_found = 0; - flow->l4.tcp.ssl_seen_server_cert = 1; + if(handshake_protocol == 0x02) + flow->l4.tcp.ssl_seen_server_cert = 1; + else + flow->l4.tcp.ssl_seen_certificate = 1; /* Check after handshake protocol header (5 bytes) and message header (4 bytes) */ for(i = 9; i < packet->payload_packet_len-3; i++) { @@ -233,7 +242,7 @@ int getSSLcertificate(struct ndpi_detection_module_struct *ndpi_struct, } } else if(handshake_protocol == 0x01 /* Client Hello */) { u_int offset, base_offset = 43; - if (base_offset + 2 <= packet->payload_packet_len) { + if(base_offset + 2 <= packet->payload_packet_len) { u_int16_t session_id_len = packet->payload[base_offset]; if((session_id_len+base_offset+2) <= total_len) { @@ -366,7 +375,7 @@ void getSSLorganization(struct ndpi_detection_module_struct *ndpi_struct, num_found++; /* what we want is subject certificate, so we bypass the issuer certificate */ - if (num_found != 2) continue; + if(num_found != 2) continue; // packet is truncated... further inspection is not needed if(i+4+server_len >= packet->payload_packet_len) { @@ -388,7 +397,7 @@ void getSSLorganization(struct ndpi_detection_module_struct *ndpi_struct, } } - if (is_printable == 1) { + if(is_printable == 1) { snprintf(flow->protos.stun_ssl.ssl.server_organization, sizeof(flow->protos.stun_ssl.ssl.server_organization), "%s", buffer); #ifdef CERTIFICATE_DEBUG @@ -411,18 +420,18 @@ int sslTryAndRetrieveServerCertificate(struct ndpi_detection_module_struct *ndpi certificate[0] = '\0'; rc = getSSLcertificate(ndpi_struct, flow, certificate, sizeof(certificate)); packet->ssl_certificate_num_checks++; - if (rc > 0) { + if(rc > 0) { // try fetch server organization once server certificate is found organization[0] = '\0'; getSSLorganization(ndpi_struct, flow, organization, sizeof(organization)); packet->ssl_certificate_detected++; - if ((flow->l4.tcp.ssl_seen_server_cert == 1) && (flow->protos.stun_ssl.ssl.server_certificate[0] != '\0')) + if((flow->l4.tcp.ssl_seen_server_cert == 1) && (flow->protos.stun_ssl.ssl.server_certificate[0] != '\0')) /* 0 means we're done processing extra packets (since we found what we wanted) */ return 0; } /* Client hello, Server Hello, and certificate packets probably all checked in this case */ - if ((packet->ssl_certificate_num_checks >= 3) + if((packet->ssl_certificate_num_checks >= 3) && (flow->l4.tcp.seen_syn) && (flow->l4.tcp.seen_syn_ack) && (flow->l4.tcp.seen_ack) /* We have seen the 3-way handshake */) @@ -438,7 +447,7 @@ int sslTryAndRetrieveServerCertificate(struct ndpi_detection_module_struct *ndpi void sslInitExtraPacketProcessing(int caseNum, struct ndpi_flow_struct *flow) { flow->check_extra_packets = 1; /* 0 is the case for waiting for the server certificate */ - if (caseNum == 0) { + if(caseNum == 0) { /* At most 7 packets should almost always be enough to find the server certificate if it's there */ flow->max_extra_packets_to_check = 7; flow->extra_packets_func = sslTryAndRetrieveServerCertificate; @@ -474,7 +483,8 @@ int sslDetectProtocolFromCertificate(struct ndpi_detection_module_struct *ndpi_s /* If we've detected the subprotocol from client certificate but haven't had a chance * to see the server certificate yet, set up extra packet processing to wait * a few more packets. */ - if(((flow->l4.tcp.ssl_seen_client_cert == 1) && (flow->protos.stun_ssl.ssl.client_certificate[0] != '\0')) && ((flow->l4.tcp.ssl_seen_server_cert != 1) && (flow->protos.stun_ssl.ssl.server_certificate[0] == '\0'))) { + if(((flow->l4.tcp.ssl_seen_client_cert == 1) && (flow->protos.stun_ssl.ssl.client_certificate[0] != '\0')) + && ((flow->l4.tcp.ssl_seen_server_cert != 1) && (flow->protos.stun_ssl.ssl.server_certificate[0] == '\0'))) { sslInitExtraPacketProcessing(0, flow); } @@ -487,11 +497,13 @@ int sslDetectProtocolFromCertificate(struct ndpi_detection_module_struct *ndpi_s return(rc); } - if(((packet->ssl_certificate_num_checks >= 2) + if(((packet->ssl_certificate_num_checks >= 3) && flow->l4.tcp.seen_syn && flow->l4.tcp.seen_syn_ack && flow->l4.tcp.seen_ack /* We have seen the 3-way handshake */) - || ((flow->l4.tcp.ssl_seen_server_cert == 1) && (flow->protos.stun_ssl.ssl.server_certificate[0] != '\0')) + || ((flow->l4.tcp.ssl_seen_certificate == 1) + && (flow->l4.tcp.ssl_seen_server_cert == 1) + && (flow->protos.stun_ssl.ssl.server_certificate[0] != '\0')) /* || ((flow->l4.tcp.ssl_seen_client_cert == 1) && (flow->protos.stun_ssl.ssl.client_certificate[0] != '\0')) */ ) { ndpi_int_ssl_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_SSL); @@ -501,8 +513,7 @@ int sslDetectProtocolFromCertificate(struct ndpi_detection_module_struct *ndpi_s return(0); } -static void ssl_mark_and_payload_search_for_other_protocols(struct - ndpi_detection_module_struct +static void ssl_mark_and_payload_search_for_other_protocols(struct ndpi_detection_module_struct *ndpi_struct, struct ndpi_flow_struct *flow) { struct ndpi_packet_struct *packet = &flow->packet; @@ -579,16 +590,16 @@ static void ssl_mark_and_payload_search_for_other_protocols(struct /* SSL without certificate (Skype, Ultrasurf?) */ NDPI_LOG_INFO(ndpi_struct, "found ssl NO_CERT\n"); ndpi_int_ssl_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_SSL_NO_CERT); - } else + } else if(packet->ssl_certificate_num_checks >= 3) { NDPI_LOG_INFO(ndpi_struct, "found ssl\n"); - ndpi_int_ssl_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_SSL); + ndpi_int_ssl_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_SSL); + } } } static u_int8_t ndpi_search_sslv3_direction1(struct ndpi_detection_module_struct *ndpi_struct, struct ndpi_flow_struct *flow) { - struct ndpi_packet_struct *packet = &flow->packet; if((packet->payload_packet_len >= 5) @@ -716,27 +727,25 @@ void ndpi_search_ssl_tcp(struct ndpi_detection_module_struct *ndpi_struct, struc NDPI_LOG_DBG(ndpi_struct, "search ssl\n"); - { - /* Check if this is whatsapp first (this proto runs over port 443) */ - if((packet->payload_packet_len > 5) - && ((packet->payload[0] == 'W') - && (packet->payload[1] == 'A') - && (packet->payload[4] == 0) - && (packet->payload[2] <= 9) - && (packet->payload[3] <= 9))) { - ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_WHATSAPP, NDPI_PROTOCOL_UNKNOWN); - return; - } else if((packet->payload_packet_len == 4) - && (packet->payload[0] == 'W') - && (packet->payload[1] == 'A')) { - ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_WHATSAPP, NDPI_PROTOCOL_UNKNOWN); + /* Check if this is whatsapp first (this proto runs over port 443) */ + if((packet->payload_packet_len > 5) + && ((packet->payload[0] == 'W') + && (packet->payload[1] == 'A') + && (packet->payload[4] == 0) + && (packet->payload[2] <= 9) + && (packet->payload[3] <= 9))) { + ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_WHATSAPP, NDPI_PROTOCOL_UNKNOWN); + return; + } else if((packet->payload_packet_len == 4) + && (packet->payload[0] == 'W') + && (packet->payload[1] == 'A')) { + ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_WHATSAPP, NDPI_PROTOCOL_UNKNOWN); + return; + } else { + /* No whatsapp, let's try SSL */ + if(sslDetectProtocolFromCertificate(ndpi_struct, flow) > 0) return; - } else { - /* No whatsapp, let's try SSL */ - if(sslDetectProtocolFromCertificate(ndpi_struct, flow) > 0) - return; - } - } + } if(packet->payload_packet_len > 40 && flow->l4.tcp.ssl_stage == 0) { NDPI_LOG_DBG2(ndpi_struct, "first ssl packet\n"); -- cgit v1.2.3 From 8dbe7818eb1755523093029255986982837290a5 Mon Sep 17 00:00:00 2001 From: Luca Deri Date: Wed, 27 Mar 2019 07:00:11 +0100 Subject: Improvments for AmazonVideo --- src/lib/ndpi_content_match.c.inc | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) (limited to 'src/lib') diff --git a/src/lib/ndpi_content_match.c.inc b/src/lib/ndpi_content_match.c.inc index 3a8a9664f..a060182d8 100644 --- a/src/lib/ndpi_content_match.c.inc +++ b/src/lib/ndpi_content_match.c.inc @@ -8029,12 +8029,13 @@ ndpi_protocol_match host_match[] = { { "dmqdd6hw24ucf.cloudfront.net", NULL, "dmqdd6hw24ucf\\.cloudfront\\.net", "AmazonVideo", NDPI_PROTOCOL_AMAZON_VIDEO, NDPI_PROTOCOL_CATEGORY_VIDEO, NDPI_PROTOCOL_FUN }, { "d25xi40x97liuc.cloudfront.net", NULL, "d25xi40x97liuc\\.cloudfront\\.net", "AmazonVideo", NDPI_PROTOCOL_AMAZON_VIDEO, NDPI_PROTOCOL_CATEGORY_VIDEO, NDPI_PROTOCOL_FUN }, { ".aiv-delivery.net", NULL, "\\.aiv-delivery\\.net", "AmazonVideo", NDPI_PROTOCOL_AMAZON_VIDEO, NDPI_PROTOCOL_CATEGORY_VIDEO, NDPI_PROTOCOL_FUN }, - { "amazon.", NULL, NULL, "Amazon", NDPI_PROTOCOL_AMAZON, NDPI_PROTOCOL_CATEGORY_WEB, NDPI_PROTOCOL_SAFE }, - { "amazon.com", NULL, "amazon" TLD, "Amazon", NDPI_PROTOCOL_AMAZON, NDPI_PROTOCOL_CATEGORY_WEB, NDPI_PROTOCOL_SAFE }, - { "images-amazon.com", NULL, "images-amazon" TLD, "Amazon", NDPI_PROTOCOL_AMAZON, NDPI_PROTOCOL_CATEGORY_WEB, NDPI_PROTOCOL_ACCEPTABLE }, - { "amazonaws.com", NULL, "amazonaws" TLD, "Amazon", NDPI_PROTOCOL_AMAZON, NDPI_PROTOCOL_CATEGORY_WEB, NDPI_PROTOCOL_ACCEPTABLE }, - { "amazon-adsystem.com", NULL, "amazon-adsystem" TLD, "Amazon", NDPI_PROTOCOL_AMAZON, NDPI_PROTOCOL_CATEGORY_WEB, NDPI_PROTOCOL_ACCEPTABLE }, - { ".cloudfront.net", NULL, "\\.cloudfront" TLD, "Amazon", NDPI_PROTOCOL_AMAZON, NDPI_PROTOCOL_CATEGORY_WEB, NDPI_PROTOCOL_ACCEPTABLE }, + { ".aiv-cdn.net", NULL, "\\.aiv-cdn\\.net", "AmazonVideo", NDPI_PROTOCOL_AMAZON_VIDEO, NDPI_PROTOCOL_CATEGORY_VIDEO, NDPI_PROTOCOL_FUN }, + { "amazon.", NULL, NULL, "Amazon", NDPI_PROTOCOL_AMAZON, NDPI_PROTOCOL_CATEGORY_WEB, NDPI_PROTOCOL_ACCEPTABLE }, + { "amazon.com", NULL, "amazon" TLD, "Amazon", NDPI_PROTOCOL_AMAZON, NDPI_PROTOCOL_CATEGORY_WEB, NDPI_PROTOCOL_ACCEPTABLE }, + { "images-amazon.com", NULL, "images-amazon" TLD, "Amazon", NDPI_PROTOCOL_AMAZON, NDPI_PROTOCOL_CATEGORY_WEB, NDPI_PROTOCOL_ACCEPTABLE }, + { "amazonaws.com", NULL, "amazonaws" TLD, "Amazon", NDPI_PROTOCOL_AMAZON, NDPI_PROTOCOL_CATEGORY_WEB, NDPI_PROTOCOL_ACCEPTABLE }, + { "amazon-adsystem.com", NULL, "amazon-adsystem" TLD, "Amazon", NDPI_PROTOCOL_AMAZON, NDPI_PROTOCOL_CATEGORY_WEB, NDPI_PROTOCOL_ACCEPTABLE }, + { ".cloudfront.net", NULL, "\\.cloudfront" TLD, "Amazon", NDPI_PROTOCOL_AMAZON, NDPI_PROTOCOL_CATEGORY_WEB, NDPI_PROTOCOL_ACCEPTABLE }, { ".push.apple.com", NULL, "\\.push\\.apple" TLD, "ApplePush", NDPI_PROTOCOL_APPLE_PUSH, NDPI_PROTOCOL_CATEGORY_CLOUD, NDPI_PROTOCOL_SAFE }, { ".apple-dns.net", NULL, "\\.apple-dns" TLD, "Apple", NDPI_PROTOCOL_APPLE, NDPI_PROTOCOL_CATEGORY_WEB, NDPI_PROTOCOL_SAFE }, -- cgit v1.2.3 From c496c7975f5d95fb4e1b9774c3592713cda78206 Mon Sep 17 00:00:00 2001 From: emanuele-f Date: Fri, 29 Mar 2019 19:58:10 +0100 Subject: Add DNS response address --- src/include/ndpi_typedefs.h | 1 + src/lib/protocols/dns.c | 16 ++++++++++++++++ 2 files changed, 17 insertions(+) (limited to 'src/lib') diff --git a/src/include/ndpi_typedefs.h b/src/include/ndpi_typedefs.h index 92c011b3d..1bd8fd2db 100644 --- a/src/include/ndpi_typedefs.h +++ b/src/include/ndpi_typedefs.h @@ -1079,6 +1079,7 @@ struct ndpi_flow_struct { struct { u_int8_t num_queries, num_answers, reply_code; u_int16_t query_type, query_class, rsp_type; + ndpi_ip_addr_t rsp_addr; /* The first address in a DNS response packet */ } dns; struct { diff --git a/src/lib/protocols/dns.c b/src/lib/protocols/dns.c index d17acd2bd..b99e5a5da 100644 --- a/src/lib/protocols/dns.c +++ b/src/lib/protocols/dns.c @@ -186,6 +186,22 @@ void ndpi_search_dns(struct ndpi_detection_module_struct *ndpi_struct, struct nd rsp_type = get16(&x, flow->packet.payload); flow->protos.dns.rsp_type = rsp_type; + + /* here x points to the response "class" field */ + if((x+12) < flow->packet.payload_packet_len) { + x += 6; + data_len = get16(&x, flow->packet.payload); + + if(((x + data_len) < flow->packet.payload_packet_len) + && (((rsp_type == 0x1) && (data_len == 4)) /* A */ +#ifdef NDPI_DETECTION_SUPPORT_IPV6 + || ((rsp_type == 0x1c) && (data_len == 16)) /* AAAA */ +#endif + )) { + memcpy(&flow->protos.dns.rsp_addr, flow->packet.payload + x, data_len); + } + } + break; } } -- cgit v1.2.3 From 1915a63cf29fbe3d2b0a983b2875929518d242ad Mon Sep 17 00:00:00 2001 From: Luca Deri Date: Mon, 1 Apr 2019 12:30:51 +0200 Subject: Implemented ndpi_process_partial_detection() API call to handle partial matches due to the nDPI specified configuration --- example/ndpiReader.c | 31 ++++----- src/include/ndpi_api.h | 15 +++- src/lib/ndpi_main.c | 176 ++++++++++++++++++++++++++++++----------------- src/lib/protocols/dns.c | 6 +- src/lib/protocols/http.c | 4 +- 5 files changed, 149 insertions(+), 83 deletions(-) (limited to 'src/lib') diff --git a/example/ndpiReader.c b/example/ndpiReader.c index 4244e3745..72eaecca4 100644 --- a/example/ndpiReader.c +++ b/example/ndpiReader.c @@ -914,11 +914,10 @@ static void node_proto_guess_walker(const void *node, ndpi_VISIT which, int dept struct ndpi_flow_info *flow = *(struct ndpi_flow_info **) node; u_int16_t thread_id = *((u_int16_t *) user_data); - if((which == ndpi_preorder) || (which == ndpi_leaf)) { /* Avoid walking the same node multiple times */ - if((!flow->detection_completed) && flow->ndpi_flow) { + if((which == ndpi_preorder) || (which == ndpi_leaf)) { /* Avoid walking the same node multiple times */ + if((!flow->detection_completed) && flow->ndpi_flow) flow->detected_protocol = ndpi_detection_giveup(ndpi_thread_info[0].workflow->ndpi_struct, flow->ndpi_flow, enable_protocol_guess); - } - + process_ndpi_collected_info(ndpi_thread_info[thread_id].workflow, flow); ndpi_thread_info[thread_id].workflow->stats.protocol_counter[flow->detected_protocol.app_protocol] += flow->src2dst_packets + flow->dst2src_packets; @@ -1956,7 +1955,7 @@ static void printResults(u_int64_t processing_time_usec, u_int64_t setup_time_us printf("\tPeak Memory: %-13s\n", formatBytes(max_ndpi_memory, buf, sizeof(buf))); printf("\tSetup Time: %lu msec\n", (unsigned long)(setup_time_usec/1000)); printf("\tPacket Processing Time: %lu msec\n", (unsigned long)(processing_time_usec/1000)); - + if(!json_flag) { printf("\nTraffic statistics:\n"); printf("\tEthernet bytes: %-13llu (includes ethernet CRC/IFC/trailer)\n", @@ -1992,10 +1991,10 @@ static void printResults(u_int64_t processing_time_usec, u_int64_t setup_time_us float t = (float)(cumulative_stats.ip_packet_count*1000000)/(float)processing_time_usec; float b = (float)(cumulative_stats.total_wire_bytes * 8 *1000000)/(float)processing_time_usec; float traffic_duration; - + if(live_capture) traffic_duration = processing_time_usec; else traffic_duration = (pcap_end.tv_sec*1000000 + pcap_end.tv_usec) - (pcap_start.tv_sec*1000000 + pcap_start.tv_usec); - + printf("\tnDPI throughput: %s pps / %s/sec\n", formatPackets(t, buf), formatTraffic(b, 1, buf1)); t = (float)(cumulative_stats.ip_packet_count*1000000)/(float)traffic_duration; b = (float)(cumulative_stats.total_wire_bytes * 8 *1000000)/(float)traffic_duration; @@ -2005,7 +2004,7 @@ static void printResults(u_int64_t processing_time_usec, u_int64_t setup_time_us strftime(when, sizeof(when), "%d/%b/%Y %H:%M:%S", localtime(&pcap_end.tv_sec)); printf("\tAnalysis end: %s\n", when); printf("\tTraffic throughput: %s pps / %s/sec\n", formatPackets(t, buf), formatTraffic(b, 1, buf1)); - printf("\tTraffic duration: %.3f sec\n", traffic_duration/1000000); + printf("\tTraffic duration: %.3f sec\n", traffic_duration/1000000); } if(enable_protocol_guess) @@ -2490,7 +2489,7 @@ static void ndpi_process_packet(u_char *args, if(memcmp(packet, packet_checked, header->caplen) != 0) printf("INTERNAL ERROR: ingress packet was modified by nDPI: this should not happen [thread_id=%u, packetId=%lu, caplen=%u]\n", thread_id, (unsigned long)ndpi_thread_info[thread_id].workflow->stats.raw_packet_count, header->caplen); - + if((pcap_end.tv_sec-pcap_start.tv_sec) > pcap_analysis_duration) { int i; u_int64_t processing_time_usec, setup_time_usec; @@ -2498,7 +2497,7 @@ static void ndpi_process_packet(u_char *args, gettimeofday(&end, NULL); processing_time_usec = end.tv_sec*1000000 + end.tv_usec - (begin.tv_sec*1000000 + begin.tv_usec); setup_time_usec = begin.tv_sec*1000000 + begin.tv_usec - (startup_time.tv_sec*1000000 + startup_time.tv_usec); - + printResults(processing_time_usec, setup_time_usec); for(i=0; iprefs.num_roots; i++) { @@ -2515,7 +2514,7 @@ static void ndpi_process_packet(u_char *args, memcpy(&pcap_start, &pcap_end, sizeof(pcap_start)); } - /* + /* Leave the free as last statement to avoid crashes when ndpi_detection_giveup() is called above by printResults() */ @@ -2541,7 +2540,7 @@ void * processing_thread(void *_thread_id) { #if defined(linux) && defined(HAVE_PTHREAD_SETAFFINITY_NP) if(core_affinity[thread_id] >= 0) { cpu_set_t cpuset; - + CPU_ZERO(&cpuset); CPU_SET(core_affinity[thread_id], &cpuset); @@ -2559,7 +2558,7 @@ void * processing_thread(void *_thread_id) { struct rte_mbuf *bufs[BURST_SIZE]; u_int16_t num = rte_eth_rx_burst(dpdk_port_id, 0, bufs, BURST_SIZE); u_int i; - + if(num == 0) { usleep(1); continue; @@ -3273,8 +3272,8 @@ int orginal_main(int argc, char **argv) { #else int main(int argc, char **argv) { #endif - int i; - + int i; + if(ndpi_get_api_version() != NDPI_API_VERSION) { printf("nDPI Library version mismatch: please make sure this code and the nDPI library are in sync\n"); return(-1); @@ -3284,7 +3283,7 @@ int orginal_main(int argc, char **argv) { gettimeofday(&startup_time, NULL); ndpi_info_mod = ndpi_init_detection_module(); - + if(ndpi_info_mod == NULL) return -1; memset(ndpi_thread_info, 0, sizeof(ndpi_thread_info)); diff --git a/src/include/ndpi_api.h b/src/include/ndpi_api.h index 386d306fd..84633c3ae 100644 --- a/src/include/ndpi_api.h +++ b/src/include/ndpi_api.h @@ -1,7 +1,7 @@ /* * ndpi_api.h * - * Copyright (C) 2011-17 - ntop.org + * Copyright (C) 2011-19 - ntop.org * * This file is part of nDPI, an open source deep packet inspection * library based on the OpenDPI and PACE technology by ipoque GmbH @@ -206,7 +206,18 @@ extern "C" { */ void ndpi_set_protocol_detection_bitmask2(struct ndpi_detection_module_struct *ndpi_struct, const NDPI_PROTOCOL_BITMASK * detection_bitmask); - + + /** + * Function to be called to see in case of unknown match to see if there is + * a partial match that has been prevented by the current nDPI preferences configuration + * + * @par ndpi_struct = the detection module + * @par flow = the flow given for the detection module + * @return the detected protocol even if the flow is not completed; + * + */ + ndpi_protocol ndpi_get_partial_detection(struct ndpi_detection_module_struct *ndpi_struct, + struct ndpi_flow_struct *flow); /** * Function to be called before we give up with detection for a given flow. * This function reduces the NDPI_UNKNOWN_PROTOCOL detection diff --git a/src/lib/ndpi_main.c b/src/lib/ndpi_main.c index 59de7a763..acbe5b8ec 100644 --- a/src/lib/ndpi_main.c +++ b/src/lib/ndpi_main.c @@ -669,7 +669,7 @@ void ndpi_set_proto_defaults(struct ndpi_detection_module_struct *ndpi_mod, ndpi_mod->proto_defaults[protoId].protoId = protoId, ndpi_mod->proto_defaults[protoId].protoBreed = breed; ndpi_mod->proto_defaults[protoId].can_have_a_subprotocol = can_have_a_subprotocol; - + memcpy(&ndpi_mod->proto_defaults[protoId].master_tcp_protoId, tcp_master_protoId, 2*sizeof(u_int16_t)); memcpy(&ndpi_mod->proto_defaults[protoId].master_udp_protoId, udp_master_protoId, 2*sizeof(u_int16_t)); @@ -677,7 +677,7 @@ void ndpi_set_proto_defaults(struct ndpi_detection_module_struct *ndpi_mod, if(udpDefPorts[j].port_low != 0) addDefaultPort(ndpi_mod, &udpDefPorts[j], &ndpi_mod->proto_defaults[protoId], 0, &ndpi_mod->udpRoot, __FUNCTION__,__LINE__); - + if(tcpDefPorts[j].port_low != 0) addDefaultPort(ndpi_mod, &tcpDefPorts[j], &ndpi_mod->proto_defaults[protoId], 0, &ndpi_mod->tcpRoot, __FUNCTION__,__LINE__); @@ -790,15 +790,15 @@ static int ndpi_string_to_automa(struct ndpi_detection_module_struct *ndpi_struc if(automa->ac_automa == NULL) return(-2); ac_pattern.astring = value, - ac_pattern.rep.number = protocol_id, + ac_pattern.rep.number = protocol_id, ac_pattern.rep.category = (u_int16_t)category, ac_pattern.rep.breed = (u_int16_t)breed; - + #ifdef MATCH_DEBUG printf("Adding to automa [%s][protocol_id: %u][category: %u][breed: %u]\n", value, protocol_id, category, breed); #endif - + if(value == NULL) ac_pattern.length = 0; else @@ -856,14 +856,14 @@ void ndpi_init_protocol_match(struct ndpi_detection_module_struct *ndpi_mod, u_int16_t no_master[2] = { NDPI_PROTOCOL_NO_MASTER_PROTO, NDPI_PROTOCOL_NO_MASTER_PROTO }; ndpi_port_range ports_a[MAX_DEFAULT_PORTS], ports_b[MAX_DEFAULT_PORTS]; static u_int16_t generic_id = NDPI_LAST_IMPLEMENTED_PROTOCOL; - + if(ndpi_mod->proto_defaults[match->protocol_id].protoName == NULL) { if(match->protocol_id == NDPI_PROTOCOL_GENERIC) ndpi_mod->proto_defaults[match->protocol_id].protoName = ndpi_strdup(NDPI_CONST_GENERIC_PROTOCOL_NAME); else ndpi_mod->proto_defaults[match->protocol_id].protoName = ndpi_strdup(match->proto_name); - - ndpi_mod->proto_defaults[match->protocol_id].protoId = match->protocol_id; + + ndpi_mod->proto_defaults[match->protocol_id].protoId = match->protocol_id; ndpi_mod->proto_defaults[match->protocol_id].protoCategory = match->protocol_category; ndpi_mod->proto_defaults[match->protocol_id].protoBreed = match->protocol_breed; @@ -875,7 +875,7 @@ void ndpi_init_protocol_match(struct ndpi_detection_module_struct *ndpi_mod, ndpi_mod->proto_defaults[match->protocol_id].protoName, ndpi_mod->proto_defaults[match->protocol_id].protoCategory, ndpi_build_default_ports(ports_a, 0, 0, 0, 0, 0) /* TCP */, - ndpi_build_default_ports(ports_b, 0, 0, 0, 0, 0) /* UDP */); + ndpi_build_default_ports(ports_b, 0, 0, 0, 0, 0) /* UDP */); } ndpi_add_host_url_subprotocol(ndpi_mod, @@ -1025,7 +1025,7 @@ static void init_string_based_protocols(struct ndpi_detection_module_struct *ndp for(i=0; host_match[i].string_to_match != NULL; i++) ndpi_init_protocol_match(ndpi_mod, &host_match[i]); -#ifdef MATCH_DEBUG +#ifdef MATCH_DEBUG // ac_automata_display(ndpi_mod->host_automa.ac_automa, 'n'); #endif @@ -1065,7 +1065,7 @@ int ndpi_set_detection_preferences(struct ndpi_detection_module_struct *ndpi_mod case ndpi_pref_enable_category_substring_match: ndpi_mod->enable_category_substring_match = (u_int8_t)value; break; - + default: return(-1); } @@ -2089,7 +2089,7 @@ static int ac_match_handler(AC_MATCH_t *m, AC_TEXT_t *txt, AC_REP_t *match) { char buf[64] = { '\0' }; int min_buf_len = (txt->length > 63 /* sizeof(buf)-1 */) ? 63 : txt->length; u_int buf_len = strlen(buf); - + strncpy(buf, txt->astring, min_buf_len); buf[min_buf_len] = '\0'; @@ -2103,8 +2103,8 @@ static int ac_match_handler(AC_MATCH_t *m, AC_TEXT_t *txt, AC_REP_t *match) { char *whatfound = strstr(buf, m->patterns->astring); #ifdef MATCH_DEBUG - printf("[NDPI] %s() [searching=%s][pattern=%s][%s][%c]\n", - __FUNCTION__, buf, m->patterns->astring, + printf("[NDPI] %s() [searching=%s][pattern=%s][%s][%c]\n", + __FUNCTION__, buf, m->patterns->astring, whatfound ? whatfound : "", whatfound[-1]); #endif @@ -2117,7 +2117,7 @@ static int ac_match_handler(AC_MATCH_t *m, AC_TEXT_t *txt, AC_REP_t *match) { if(whatfound && (whatfound != buf) && (m->patterns->astring[0] != '.') /* The searched patter does not start with . */ && strchr(m->patterns->astring, '.') /* The matched pattern has a . (e.g. numeric or sym IPs) */ - && (whatfound[-1] != '.') + && (whatfound[-1] != '.') ) return(0); } @@ -2368,7 +2368,7 @@ struct ndpi_detection_module_struct *ndpi_init_detection_module(void) { #endif ndpi_str->custom_categories.hostnames_hash = NULL; - + ndpi_str->custom_categories.ipAddresses = ndpi_New_Patricia(32 /* IPv4 */); ndpi_str->custom_categories.ipAddresses_shadow = ndpi_New_Patricia(32 /* IPv4 */); @@ -2436,7 +2436,7 @@ int ndpi_match_string(void *_automa, char *string_to_match) { int ndpi_match_string_id(void *_automa, char *string_to_match, unsigned long *id) { AC_TEXT_t ac_input_text; AC_AUTOMATA_t *automa = (AC_AUTOMATA_t*)_automa; - AC_REP_t match = { NDPI_PROTOCOL_UNKNOWN, NDPI_PROTOCOL_CATEGORY_UNSPECIFIED, NDPI_PROTOCOL_UNRATED }; + AC_REP_t match = { NDPI_PROTOCOL_UNKNOWN, NDPI_PROTOCOL_CATEGORY_UNSPECIFIED, NDPI_PROTOCOL_UNRATED }; *id = -1; if((automa == NULL) @@ -2449,7 +2449,7 @@ int ndpi_match_string_id(void *_automa, char *string_to_match, unsigned long *id ac_automata_reset(automa); *id = match.number; - + return(*id != NDPI_PROTOCOL_UNKNOWN ? 0 : -1); } @@ -2476,7 +2476,7 @@ static int hyperscanCustomEventHandler(unsigned int id, int ndpi_match_custom_category(struct ndpi_detection_module_struct *ndpi_struct, char *name, unsigned long *id) { #ifdef DEBUG - printf("[NDPI] %s(%s) [enable_category_substring_match: %u]\n", + printf("[NDPI] %s(%s) [enable_category_substring_match: %u]\n", __FUNCTION__, name, ndpi_struct->enable_category_substring_match); #endif @@ -2575,7 +2575,7 @@ void ndpi_exit_detection_module(struct ndpi_detection_module_struct *ndpi_struct if(ndpi_struct->protocols_ptree) ndpi_Destroy_Patricia((patricia_tree_t*)ndpi_struct->protocols_ptree, free_ptree_data); - + if(ndpi_struct->udpRoot != NULL) ndpi_tdestroy(ndpi_struct->udpRoot, ndpi_free); if(ndpi_struct->tcpRoot != NULL) @@ -2621,7 +2621,7 @@ void ndpi_exit_detection_module(struct ndpi_detection_module_struct *ndpi_struct if(ndpi_struct->custom_categories.hostnames_hash) ht_free((hashtable_t*)ndpi_struct->custom_categories.hostnames_hash); - + ndpi_free(ndpi_struct); } } @@ -2634,7 +2634,7 @@ int ndpi_get_protocol_id_master_proto(struct ndpi_detection_module_struct *ndpi_ u_int16_t** udp_master_proto) { if(protocol_id >= (NDPI_MAX_SUPPORTED_PROTOCOLS+NDPI_MAX_NUM_CUSTOM_PROTOCOLS)) { *tcp_master_proto = ndpi_struct->proto_defaults[NDPI_PROTOCOL_UNKNOWN].master_tcp_protoId, - *udp_master_proto = ndpi_struct->proto_defaults[NDPI_PROTOCOL_UNKNOWN].master_udp_protoId; + *udp_master_proto = ndpi_struct->proto_defaults[NDPI_PROTOCOL_UNKNOWN].master_udp_protoId; return(-1); } @@ -2782,7 +2782,7 @@ char * strsep(char **sp, char *sep) /* ******************************************************************** */ -int ndpi_handle_rule(struct ndpi_detection_module_struct *ndpi_mod, +int ndpi_handle_rule(struct ndpi_detection_module_struct *ndpi_mod, char* rule, u_int8_t do_add) { char *at, *proto, *elem; ndpi_proto_defaults_t *def; @@ -2917,7 +2917,7 @@ int ndpi_load_protocols_file(struct ndpi_detection_module_struct *ndpi_mod, char if(buffer == NULL) { NDPI_LOG_ERR(ndpi_mod, "Memory allocation failure"); - goto close_fd; + goto close_fd; } while(fd) { @@ -3820,18 +3820,18 @@ static int ndpi_init_packet_header(struct ndpi_detection_module_struct *ndpi_str && flow->detected_protocol_stack[0] == NDPI_PROTOCOL_UNKNOWN) { u_int8_t backup; u_int16_t backup1, backup2; - + if(flow->http.url) ndpi_free(flow->http.url); if(flow->http.content_type) ndpi_free(flow->http.content_type); backup = flow->num_processed_pkts; backup1 = flow->guessed_protocol_id; - backup2 = flow->guessed_host_protocol_id; + backup2 = flow->guessed_host_protocol_id; memset(flow, 0, sizeof(*(flow))); flow->num_processed_pkts = backup; flow->guessed_protocol_id = backup1; flow->guessed_host_protocol_id = backup2; - + NDPI_LOG_DBG(ndpi_struct, "tcp syn packet for unknown protocol, reset detection state\n"); } @@ -4165,6 +4165,52 @@ static u_int16_t ndpi_guess_host_protocol_id(struct ndpi_detection_module_struct /* ********************************************************************************* */ +static ndpi_protocol ndpi_process_partial_detection(struct ndpi_detection_module_struct *ndpi_struct, + struct ndpi_flow_struct *flow) { + ndpi_protocol ret; + ndpi_protocol_match_result ret_match; + + ret.master_protocol = flow->guessed_protocol_id; + ret.app_protocol = ndpi_match_host_subprotocol(ndpi_struct, flow, + (char *)flow->host_server_name, + strlen((const char*)flow->host_server_name), + &ret_match, + flow->guessed_protocol_id); + ret.category = ret_match.protocol_category; + + if(ret.app_protocol == NDPI_PROTOCOL_UNKNOWN) + ret.app_protocol = ret.master_protocol; + + ndpi_int_change_protocol(ndpi_struct, flow, ret.app_protocol, ret.master_protocol); + return(ret); +} + +/* ********************************************************************************* */ + +/* + You can call this function at any time in case of unknown match to see if there is + a partial match that has been prevented by the current nDPI preferences configuration + */ +ndpi_protocol ndpi_get_partial_detection(struct ndpi_detection_module_struct *ndpi_struct, + struct ndpi_flow_struct *flow) { + if((flow->guessed_protocol_id == NDPI_PROTOCOL_HTTP) + && (ndpi_struct->http_dont_dissect_response == 0) + && (flow->host_server_name[0] != '\0') + && (!NDPI_ISSET(&flow->excluded_protocol_bitmask, flow->guessed_host_protocol_id))) + return(ndpi_process_partial_detection(ndpi_struct, flow)); + else if((flow->guessed_protocol_id == NDPI_PROTOCOL_DNS) + && (ndpi_struct->dns_dont_dissect_response == 0) + && (flow->host_server_name[0] != '\0') + && (!NDPI_ISSET(&flow->excluded_protocol_bitmask, flow->guessed_host_protocol_id))) + return(ndpi_process_partial_detection(ndpi_struct, flow)); + else { + ndpi_protocol ret = { NDPI_PROTOCOL_UNKNOWN, NDPI_PROTOCOL_UNKNOWN, NDPI_PROTOCOL_CATEGORY_UNSPECIFIED }; + return(ret); + } +} + +/* ********************************************************************************* */ + ndpi_protocol ndpi_detection_giveup(struct ndpi_detection_module_struct *ndpi_struct, struct ndpi_flow_struct *flow, u_int8_t enable_guess) { ndpi_protocol ret = { NDPI_PROTOCOL_UNKNOWN, NDPI_PROTOCOL_UNKNOWN, NDPI_PROTOCOL_CATEGORY_UNSPECIFIED }; @@ -4177,12 +4223,19 @@ ndpi_protocol ndpi_detection_giveup(struct ndpi_detection_module_struct *ndpi_st if(flow->guessed_protocol_id == NDPI_PROTOCOL_STUN) goto check_stun_export; - else if((flow->l4.tcp.ssl_seen_client_cert == 1) && (flow->protos.stun_ssl.ssl.client_certificate[0] != '\0')) { + else if((flow->l4.tcp.ssl_seen_client_cert == 1) + && (flow->protos.stun_ssl.ssl.client_certificate[0] != '\0')) { ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_SSL, NDPI_PROTOCOL_UNKNOWN); } else { - if(!enable_guess) - return(ret); - + ndpi_protocol ret_g = ndpi_get_partial_detection(ndpi_struct, flow); + + if(ret_g.master_protocol != NDPI_PROTOCOL_UNKNOWN) + return(ret_g); + else { + if(!enable_guess) + return(ret); + } + if((flow->guessed_protocol_id == NDPI_PROTOCOL_UNKNOWN) && (flow->packet.l4_protocol == IPPROTO_TCP) && (flow->l4.tcp.ssl_stage > 1)) @@ -4196,7 +4249,7 @@ ndpi_protocol ndpi_detection_giveup(struct ndpi_detection_module_struct *ndpi_st && is_udp_guessable_protocol(guessed_host_protocol_id) )) flow->guessed_host_protocol_id = guessed_host_protocol_id = NDPI_PROTOCOL_UNKNOWN; - + /* Ignore guessed protocol if they have been discarded */ if((guessed_protocol_id != NDPI_PROTOCOL_UNKNOWN) // && (guessed_host_protocol_id == NDPI_PROTOCOL_UNKNOWN) @@ -4207,7 +4260,6 @@ ndpi_protocol ndpi_detection_giveup(struct ndpi_detection_module_struct *ndpi_st if((guessed_protocol_id != NDPI_PROTOCOL_UNKNOWN) || (guessed_host_protocol_id != NDPI_PROTOCOL_UNKNOWN)) { - if((guessed_protocol_id == 0) && (flow->protos.stun_ssl.stun.num_binding_requests > 0) && (flow->protos.stun_ssl.stun.num_processed_pkts > 0)) @@ -4246,7 +4298,7 @@ ndpi_protocol ndpi_detection_giveup(struct ndpi_detection_module_struct *ndpi_st else if(ret.app_protocol == NDPI_PROTOCOL_GOOGLE) ret.app_protocol = NDPI_PROTOCOL_HANGOUT; } - + if(enable_guess && (ret.app_protocol == NDPI_PROTOCOL_UNKNOWN) && flow->packet.iph /* Guess only IPv4 */ @@ -4260,7 +4312,7 @@ ndpi_protocol ndpi_detection_giveup(struct ndpi_detection_module_struct *ndpi_st ntohl(flow->packet.iph->daddr), ntohs(flow->packet.udp ? flow->packet.udp->dest : flow->packet.tcp->dest) ); - + ndpi_fill_protocol_category(ndpi_struct, flow, &ret); return(ret); @@ -4353,15 +4405,15 @@ int ndpi_load_hostname_category(struct ndpi_detection_module_struct *ndpi_struct if(ndpi_struct->custom_categories.hostnames_hash) ht_set((hashtable_t*)ndpi_struct->custom_categories.hostnames_hash, name, (u_int16_t)category); - - return(0); + + return(0); } else { AC_PATTERN_t ac_pattern; - + /* printf("===> Loading %s as %u\n", name, category); */ memset(&ac_pattern, 0, sizeof(ac_pattern)); - + #ifdef HAVE_HYPERSCAN { struct hs_list *h = (struct hs_list*)malloc(sizeof(struct hs_list)); @@ -4401,7 +4453,7 @@ int ndpi_load_hostname_category(struct ndpi_detection_module_struct *ndpi_struct ac_automata_add(ndpi_struct->custom_categories.hostnames_shadow.ac_automa, &ac_pattern); #endif } - + return(0); } @@ -4481,16 +4533,16 @@ int ndpi_enable_loaded_categories(struct ndpi_detection_module_struct *ndpi_str) ndpi_str->custom_categories.hostnames_shadow.ac_automa = ac_automata_init(ac_match_handler); #endif } - + if(ndpi_str->custom_categories.ipAddresses != NULL) - ndpi_Destroy_Patricia((patricia_tree_t*)ndpi_str->custom_categories.ipAddresses, + ndpi_Destroy_Patricia((patricia_tree_t*)ndpi_str->custom_categories.ipAddresses, free_ptree_data); ndpi_str->custom_categories.ipAddresses = ndpi_str->custom_categories.ipAddresses_shadow; ndpi_str->custom_categories.ipAddresses_shadow = ndpi_New_Patricia(32 /* IPv4 */); ndpi_str->custom_categories.categories_loaded = 1; - + return(0); } @@ -4582,7 +4634,7 @@ ndpi_protocol ndpi_detection_process_packet(struct ndpi_detection_module_struct return(ret); flow->num_processed_pkts++; - + if(flow->server_id == NULL) flow->server_id = dst; /* Default */ if(flow->detected_protocol_stack[0] != NDPI_PROTOCOL_UNKNOWN) goto ret_protocols; @@ -4705,7 +4757,7 @@ ndpi_protocol ndpi_detection_process_packet(struct ndpi_detection_module_struct break; } } - + ndpi_check_flow_func(ndpi_struct, flow, &ndpi_selection_packet); ndpi_fill_protocol_category(ndpi_struct, flow, &ret); @@ -4969,22 +5021,22 @@ void ndpi_parse_packet_line_info(struct ndpi_detection_module_struct *ndpi_struc packet->http_response.ptr = &packet->line[0].ptr[NDPI_STATICSTRING_LEN("HTTP/1.1 ")]; packet->http_response.len = packet->line[0].len - NDPI_STATICSTRING_LEN("HTTP/1.1 "); packet->http_num_headers++; - + /* Set server HTTP response code */ if(packet->payload_packet_len >= 12) { char buf[4]; - + /* Set server HTTP response code */ strncpy(buf, (char*)&packet->payload[9], 3); buf[3] = '\0'; - + flow->http.response_status_code = atoi(buf); /* https://en.wikipedia.org/wiki/List_of_HTTP_status_codes */ if((flow->http.response_status_code < 100) || (flow->http.response_status_code > 509)) flow->http.response_status_code = 0; /* Out of range */ } } - + /* "Server:" header line in HTTP response */ if(packet->line[packet->parsed_lines].len > NDPI_STATICSTRING_LEN("Server:") + 1 && strncasecmp((const char *)packet->line[packet->parsed_lines].ptr, "Server:", NDPI_STATICSTRING_LEN("Server:")) == 0) { @@ -5403,7 +5455,7 @@ void ndpi_int_change_protocol(struct ndpi_detection_module_struct *ndpi_struct, } } } - + ndpi_int_change_flow_protocol(ndpi_struct, flow, upper_detected_protocol, lower_detected_protocol); ndpi_int_change_packet_protocol(ndpi_struct, flow, @@ -5435,7 +5487,7 @@ void ndpi_int_reset_protocol(struct ndpi_flow_struct *flow) { int a; for(a = 0; a < NDPI_PROTOCOL_SIZE; a++) - flow->detected_protocol_stack[a] = NDPI_PROTOCOL_UNKNOWN; + flow->detected_protocol_stack[a] = NDPI_PROTOCOL_UNKNOWN; } } @@ -5657,10 +5709,10 @@ ndpi_protocol ndpi_guess_undetected_protocol(struct ndpi_detection_module_struct ret.app_protocol = rc, ret.master_protocol = ndpi_guess_protocol_id(ndpi_struct, NULL, proto, sport, dport, &user_defined_proto); - + if(ret.app_protocol == ret.master_protocol) ret.master_protocol = NDPI_PROTOCOL_UNKNOWN; - + ret.category = ndpi_get_proto_category(ndpi_struct, ret); return(ret); } @@ -5674,7 +5726,7 @@ ndpi_protocol ndpi_guess_undetected_protocol(struct ndpi_detection_module_struct ; else { ret.app_protocol = rc; - + if(rc == NDPI_PROTOCOL_SSL) goto check_guessed_skype; else { @@ -5697,7 +5749,7 @@ ndpi_protocol ndpi_guess_undetected_protocol(struct ndpi_detection_module_struct ret.app_protocol = ndpi_guess_protocol_id(ndpi_struct, NULL, proto, sport, dport, &user_defined_proto); - ret.category = ndpi_get_proto_category(ndpi_struct, ret); + ret.category = ndpi_get_proto_category(ndpi_struct, ret); return(ret); } @@ -5939,7 +5991,7 @@ char* ndpi_strnstr(const char *s, const char *find, size_t slen) { } while(strncmp(s, find, len) != 0); s--; } - + return((char *)s); } @@ -5987,7 +6039,7 @@ int ndpi_match_string_subprotocol(struct ndpi_detection_module_struct *ndpi_stru ndpi_automa *automa = is_host_match ? &ndpi_struct->host_automa : &ndpi_struct->content_automa; AC_REP_t match = { NDPI_PROTOCOL_UNKNOWN, NDPI_PROTOCOL_CATEGORY_UNSPECIFIED, NDPI_PROTOCOL_UNRATED }; - + if((automa->ac_automa == NULL) || (string_to_match_len == 0)) return(NDPI_PROTOCOL_UNKNOWN); @@ -5995,15 +6047,15 @@ int ndpi_match_string_subprotocol(struct ndpi_detection_module_struct *ndpi_stru ac_automata_finalize((AC_AUTOMATA_t*)automa->ac_automa); automa->ac_automa_finalized = 1; } - + ac_input_text.astring = string_to_match, ac_input_text.length = string_to_match_len; ac_automata_search(((AC_AUTOMATA_t*)automa->ac_automa), &ac_input_text, &match); ac_automata_reset(((AC_AUTOMATA_t*)automa->ac_automa)); - + ret_match->protocol_id = match.number, ret_match->protocol_category = match.category, ret_match->protocol_breed = match.breed; - + return(match.number); } @@ -6252,9 +6304,9 @@ struct ndpi_lru_cache* ndpi_lru_cache_init(u_int32_t num_entries) { if(!c->entries) { free(c); return(NULL); - } else + } else c->num_entries = num_entries; - + return(c); } diff --git a/src/lib/protocols/dns.c b/src/lib/protocols/dns.c index b99e5a5da..1c2593feb 100644 --- a/src/lib/protocols/dns.c +++ b/src/lib/protocols/dns.c @@ -153,8 +153,10 @@ void ndpi_search_dns(struct ndpi_detection_module_struct *ndpi_struct, struct nd || ((dns_header.authority_rrs > 0) && (dns_header.authority_rrs <= NDPI_MAX_DNS_REQUESTS)) || ((dns_header.additional_rrs > 0) && (dns_header.additional_rrs <= NDPI_MAX_DNS_REQUESTS))) ) { - /* This is a good reply */ - if(ndpi_struct->dns_dont_dissect_response == 0) { + /* This is a good reply: we dissect it both for request and response */ + + /* Leave the statement below commented necessary in case of call to ndpi_get_partial_detection() */ + /* if(ndpi_struct->dns_dont_dissect_response == 0) */ { x++; if(flow->packet.payload[x] != '\0') { diff --git a/src/lib/protocols/http.c b/src/lib/protocols/http.c index fc392c2b7..33ef9e2ed 100644 --- a/src/lib/protocols/http.c +++ b/src/lib/protocols/http.c @@ -157,7 +157,9 @@ static void check_content_type_and_change_protocol(struct ndpi_detection_module_ } #endif - if(!ndpi_struct->http_dont_dissect_response) { + /* Leave the statement below commented necessary in case of call to ndpi_get_partial_detection() */ + + /* if(!ndpi_struct->http_dont_dissect_response) */ { if((flow->http.url == NULL) && (packet->http_url_name.len > 0) && (packet->host_line.len > 0)) { -- cgit v1.2.3 From 6693151052a98e6eddc722c139886d7fe84f35e4 Mon Sep 17 00:00:00 2001 From: Luca Deri Date: Mon, 1 Apr 2019 16:12:44 +0200 Subject: Added custom category support to ndpi_get_partial_detection() --- example/ndpiReader.c | 3 ++- src/lib/ndpi_main.c | 8 ++++---- 2 files changed, 6 insertions(+), 5 deletions(-) (limited to 'src/lib') diff --git a/example/ndpiReader.c b/example/ndpiReader.c index 72eaecca4..0b0fa889b 100644 --- a/example/ndpiReader.c +++ b/example/ndpiReader.c @@ -1505,7 +1505,8 @@ static void setupDetection(u_int16_t thread_id, pcap_t * pcap_handle) { if(category) { int fields[4]; - // printf("Loading %s\t%s\n", name, category); + + if(verbose) printf("[Category] Loading %s\t%s\n", name, category); if(sscanf(name, "%d.%d.%d.%d", &fields[0], &fields[1], &fields[2], &fields[3]) == 4) ndpi_load_ip_category(ndpi_thread_info[thread_id].workflow->ndpi_struct, diff --git a/src/lib/ndpi_main.c b/src/lib/ndpi_main.c index acbe5b8ec..cc5637cbc 100644 --- a/src/lib/ndpi_main.c +++ b/src/lib/ndpi_main.c @@ -2474,7 +2474,7 @@ static int hyperscanCustomEventHandler(unsigned int id, /* *********************************************** */ int ndpi_match_custom_category(struct ndpi_detection_module_struct *ndpi_struct, - char *name, unsigned long *id) { + char *name, unsigned long *id) { #ifdef DEBUG printf("[NDPI] %s(%s) [enable_category_substring_match: %u]\n", __FUNCTION__, name, ndpi_struct->enable_category_substring_match); @@ -2518,7 +2518,7 @@ int ndpi_match_custom_category(struct ndpi_detection_module_struct *ndpi_struct, /* *********************************************** */ int ndpi_get_custom_category_match(struct ndpi_detection_module_struct *ndpi_struct, - char *name_or_ip, unsigned long *id) { + char *name_or_ip, unsigned long *id) { char ipbuf[64]; struct in_addr pin; @@ -4180,7 +4180,8 @@ static ndpi_protocol ndpi_process_partial_detection(struct ndpi_detection_module if(ret.app_protocol == NDPI_PROTOCOL_UNKNOWN) ret.app_protocol = ret.master_protocol; - + + ndpi_fill_protocol_category(ndpi_struct, flow, &ret); ndpi_int_change_protocol(ndpi_struct, flow, ret.app_protocol, ret.master_protocol); return(ret); } @@ -6090,7 +6091,6 @@ static int ndpi_automa_match_string_subprotocol(struct ndpi_detection_module_str matching_protocol_id = ndpi_match_string_subprotocol(ndpi_struct, string_to_match, string_to_match_len, ret_match, is_host_match); - #else struct hs *hs = (struct hs*)ndpi_struct->hyperscan; hs_error_t status; -- cgit v1.2.3 From 153c77c2cd28d52d6b459263dea3ce988ceccd3c Mon Sep 17 00:00:00 2001 From: Luca Deri Date: Mon, 1 Apr 2019 18:40:14 +0200 Subject: Improvements with category detection --- src/lib/ndpi_main.c | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) (limited to 'src/lib') diff --git a/src/lib/ndpi_main.c b/src/lib/ndpi_main.c index cc5637cbc..866f65a10 100644 --- a/src/lib/ndpi_main.c +++ b/src/lib/ndpi_main.c @@ -4176,13 +4176,18 @@ static ndpi_protocol ndpi_process_partial_detection(struct ndpi_detection_module strlen((const char*)flow->host_server_name), &ret_match, flow->guessed_protocol_id); - ret.category = ret_match.protocol_category; + if(flow->category != NDPI_PROTOCOL_CATEGORY_UNSPECIFIED) + ret.category = flow->category; + else + ret.category = ret_match.protocol_category; + if(ret.app_protocol == NDPI_PROTOCOL_UNKNOWN) ret.app_protocol = ret.master_protocol; ndpi_fill_protocol_category(ndpi_struct, flow, &ret); ndpi_int_change_protocol(ndpi_struct, flow, ret.app_protocol, ret.master_protocol); + return(ret); } @@ -4205,7 +4210,12 @@ ndpi_protocol ndpi_get_partial_detection(struct ndpi_detection_module_struct *nd && (!NDPI_ISSET(&flow->excluded_protocol_bitmask, flow->guessed_host_protocol_id))) return(ndpi_process_partial_detection(ndpi_struct, flow)); else { - ndpi_protocol ret = { NDPI_PROTOCOL_UNKNOWN, NDPI_PROTOCOL_UNKNOWN, NDPI_PROTOCOL_CATEGORY_UNSPECIFIED }; + ndpi_protocol ret = { NDPI_PROTOCOL_UNKNOWN, + NDPI_PROTOCOL_UNKNOWN, + NDPI_PROTOCOL_CATEGORY_UNSPECIFIED }; + + if(flow) ret.category = flow->category; + return(ret); } } @@ -4216,8 +4226,11 @@ ndpi_protocol ndpi_detection_giveup(struct ndpi_detection_module_struct *ndpi_st struct ndpi_flow_struct *flow, u_int8_t enable_guess) { ndpi_protocol ret = { NDPI_PROTOCOL_UNKNOWN, NDPI_PROTOCOL_UNKNOWN, NDPI_PROTOCOL_CATEGORY_UNSPECIFIED }; - if(flow == NULL) return(ret); - + if(flow == NULL) + return(ret); + else + ret.category = flow->category; + /* TODO: add the remaining stage_XXXX protocols */ if(flow->detected_protocol_stack[0] == NDPI_PROTOCOL_UNKNOWN) { u_int16_t guessed_protocol_id, guessed_host_protocol_id; @@ -4631,9 +4644,12 @@ ndpi_protocol ndpi_detection_process_packet(struct ndpi_detection_module_struct if(ndpi_struct->ndpi_log_level >= NDPI_LOG_TRACE) NDPI_LOG(flow ? flow->detected_protocol_stack[0]:NDPI_PROTOCOL_UNKNOWN, ndpi_struct, NDPI_LOG_TRACE, "START packet processing\n"); + if(flow == NULL) return(ret); - + else + ret.category = flow->category; + flow->num_processed_pkts++; if(flow->server_id == NULL) flow->server_id = dst; /* Default */ -- cgit v1.2.3