diff options
author | Luca Deri <deri@ntop.org> | 2019-01-07 00:28:29 +0100 |
---|---|---|
committer | Luca Deri <deri@ntop.org> | 2019-01-07 00:28:29 +0100 |
commit | 97bdfe295d3f7318d1eac3e0020b3b13004f008d (patch) | |
tree | 20bd0f348beba9149460e00631e78ed5880501fe | |
parent | 5241c9f3cfefd8da67c2e07de1864ac956f9788d (diff) |
nDPi now finally honours dissection of HTTP responses
-rw-r--r-- | src/include/ndpi_typedefs.h | 443 | ||||
-rw-r--r-- | src/lib/ndpi_main.c | 43 | ||||
-rw-r--r-- | src/lib/protocols/apple_push.c | 30 | ||||
-rw-r--r-- | src/lib/protocols/http.c | 263 |
4 files changed, 395 insertions, 384 deletions
diff --git a/src/include/ndpi_typedefs.h b/src/include/ndpi_typedefs.h index 294af22b3..34d308e2e 100644 --- a/src/include/ndpi_typedefs.h +++ b/src/include/ndpi_typedefs.h @@ -29,18 +29,18 @@ /* NDPI_LOG_LEVEL */ typedef enum { - NDPI_LOG_ERROR, - NDPI_LOG_TRACE, - NDPI_LOG_DEBUG, - NDPI_LOG_DEBUG_EXTRA + NDPI_LOG_ERROR, + NDPI_LOG_TRACE, + NDPI_LOG_DEBUG, + NDPI_LOG_DEBUG_EXTRA } ndpi_log_level_t; /* NDPI_VISIT */ typedef enum { - ndpi_preorder, - ndpi_postorder, - ndpi_endorder, - ndpi_leaf + ndpi_preorder, + ndpi_postorder, + ndpi_endorder, + ndpi_leaf } ndpi_VISIT; /* NDPI_NODE */ @@ -356,15 +356,15 @@ PACK_ON struct tinc_cache_entry { } PACK_OFF; typedef enum { - HTTP_METHOD_UNKNOWN = 0, - HTTP_METHOD_OPTIONS, - HTTP_METHOD_GET, - HTTP_METHOD_HEAD, - HTTP_METHOD_POST, - HTTP_METHOD_PUT, - HTTP_METHOD_DELETE, - HTTP_METHOD_TRACE, - HTTP_METHOD_CONNECT + HTTP_METHOD_UNKNOWN = 0, + HTTP_METHOD_OPTIONS, + HTTP_METHOD_GET, + HTTP_METHOD_HEAD, + HTTP_METHOD_POST, + HTTP_METHOD_PUT, + HTTP_METHOD_DELETE, + HTTP_METHOD_TRACE, + HTTP_METHOD_CONNECT } ndpi_http_method; struct ndpi_id_struct { @@ -375,229 +375,229 @@ struct ndpi_id_struct { to compare this, use: **/ NDPI_PROTOCOL_BITMASK detected_protocol_bitmask; -/* NDPI_PROTOCOL_RTSP */ + /* NDPI_PROTOCOL_RTSP */ ndpi_ip_addr_t rtsp_ip_address; -/* NDPI_PROTOCOL_YAHOO */ + /* NDPI_PROTOCOL_YAHOO */ u_int32_t yahoo_video_lan_timer; /* NDPI_PROTOCOL_IRC_MAXPORT % 2 must be 0 */ -/* NDPI_PROTOCOL_IRC */ + /* NDPI_PROTOCOL_IRC */ #define NDPI_PROTOCOL_IRC_MAXPORT 8 u_int16_t irc_port[NDPI_PROTOCOL_IRC_MAXPORT]; u_int32_t last_time_port_used[NDPI_PROTOCOL_IRC_MAXPORT]; u_int32_t irc_ts; -/* NDPI_PROTOCOL_GNUTELLA */ + /* NDPI_PROTOCOL_GNUTELLA */ u_int32_t gnutella_ts; -/* NDPI_PROTOCOL_BATTLEFIELD */ + /* NDPI_PROTOCOL_BATTLEFIELD */ u_int32_t battlefield_ts; -/* NDPI_PROTOCOL_THUNDER */ + /* NDPI_PROTOCOL_THUNDER */ u_int32_t thunder_ts; -/* NDPI_PROTOCOL_RTSP */ + /* NDPI_PROTOCOL_RTSP */ u_int32_t rtsp_timer; -/* NDPI_PROTOCOL_OSCAR */ + /* NDPI_PROTOCOL_OSCAR */ u_int32_t oscar_last_safe_access_time; -/* NDPI_PROTOCOL_ZATTOO */ + /* NDPI_PROTOCOL_ZATTOO */ u_int32_t zattoo_ts; -/* NDPI_PROTOCOL_UNENCRYPTED_JABBER */ + /* NDPI_PROTOCOL_UNENCRYPTED_JABBER */ u_int32_t jabber_stun_or_ft_ts; -/* NDPI_PROTOCOL_DIRECTCONNECT */ + /* NDPI_PROTOCOL_DIRECTCONNECT */ u_int32_t directconnect_last_safe_access_time; -/* NDPI_PROTOCOL_SOULSEEK */ + /* NDPI_PROTOCOL_SOULSEEK */ u_int32_t soulseek_last_safe_access_time; -/* NDPI_PROTOCOL_DIRECTCONNECT */ + /* NDPI_PROTOCOL_DIRECTCONNECT */ u_int16_t detected_directconnect_port; u_int16_t detected_directconnect_udp_port; u_int16_t detected_directconnect_ssl_port; -/* NDPI_PROTOCOL_BITTORRENT */ + /* NDPI_PROTOCOL_BITTORRENT */ #define NDPI_BT_PORTS 8 u_int16_t bt_port_t[NDPI_BT_PORTS]; u_int16_t bt_port_u[NDPI_BT_PORTS]; -/* NDPI_PROTOCOL_UNENCRYPTED_JABBER */ + /* NDPI_PROTOCOL_UNENCRYPTED_JABBER */ #define JABBER_MAX_STUN_PORTS 6 u_int16_t jabber_voice_stun_port[JABBER_MAX_STUN_PORTS]; u_int16_t jabber_file_transfer_port[2]; -/* NDPI_PROTOCOL_GNUTELLA */ + /* NDPI_PROTOCOL_GNUTELLA */ u_int16_t detected_gnutella_port; -/* NDPI_PROTOCOL_GNUTELLA */ + /* NDPI_PROTOCOL_GNUTELLA */ u_int16_t detected_gnutella_udp_port1; u_int16_t detected_gnutella_udp_port2; -/* NDPI_PROTOCOL_SOULSEEK */ + /* NDPI_PROTOCOL_SOULSEEK */ u_int16_t soulseek_listen_port; -/* NDPI_PROTOCOL_IRC */ + /* NDPI_PROTOCOL_IRC */ u_int8_t irc_number_of_port; -/* NDPI_PROTOCOL_OSCAR */ + /* NDPI_PROTOCOL_OSCAR */ u_int8_t oscar_ssl_session_id[33]; -/* NDPI_PROTOCOL_UNENCRYPTED_JABBER */ + /* NDPI_PROTOCOL_UNENCRYPTED_JABBER */ u_int8_t jabber_voice_stun_used_ports; -/* NDPI_PROTOCOL_SIP */ -/* NDPI_PROTOCOL_YAHOO */ + /* NDPI_PROTOCOL_SIP */ + /* NDPI_PROTOCOL_YAHOO */ u_int32_t yahoo_video_lan_dir:1; -/* NDPI_PROTOCOL_YAHOO */ + /* NDPI_PROTOCOL_YAHOO */ u_int32_t yahoo_conf_logged_in:1; u_int32_t yahoo_voice_conf_logged_in:1; -/* NDPI_PROTOCOL_RTSP */ + /* NDPI_PROTOCOL_RTSP */ u_int32_t rtsp_ts_set:1; }; /* ************************************************** */ struct ndpi_flow_tcp_struct { -/* NDPI_PROTOCOL_MAIL_SMTP */ + /* NDPI_PROTOCOL_MAIL_SMTP */ u_int16_t smtp_command_bitmask; -/* NDPI_PROTOCOL_MAIL_POP */ + /* NDPI_PROTOCOL_MAIL_POP */ u_int16_t pop_command_bitmask; -/* NDPI_PROTOCOL_QQ */ + /* NDPI_PROTOCOL_QQ */ u_int16_t qq_nxt_len; /* NDPI_PROTOCOL_WHATSAPP */ u_int8_t wa_matched_so_far; -/* NDPI_PROTOCOL_TDS */ + /* NDPI_PROTOCOL_TDS */ u_int8_t tds_login_version; -/* NDPI_PROTOCOL_IRC */ + /* NDPI_PROTOCOL_IRC */ u_int8_t irc_stage; u_int8_t irc_port; -/* NDPI_PROTOCOL_H323 */ + /* NDPI_PROTOCOL_H323 */ u_int8_t h323_valid_packets; -/* NDPI_PROTOCOL_GNUTELLA */ + /* NDPI_PROTOCOL_GNUTELLA */ u_int8_t gnutella_msg_id[3]; -/* NDPI_PROTOCOL_IRC */ + /* NDPI_PROTOCOL_IRC */ u_int32_t irc_3a_counter:3; u_int32_t irc_stage2:5; u_int32_t irc_direction:2; u_int32_t irc_0x1000_full:1; -/* NDPI_PROTOCOL_SOULSEEK */ + /* NDPI_PROTOCOL_SOULSEEK */ u_int32_t soulseek_stage:2; -/* NDPI_PROTOCOL_TDS */ + /* NDPI_PROTOCOL_TDS */ u_int32_t tds_stage:3; -/* NDPI_PROTOCOL_USENET */ + /* NDPI_PROTOCOL_USENET */ u_int32_t usenet_stage:2; -/* NDPI_PROTOCOL_IMESH */ + /* NDPI_PROTOCOL_IMESH */ u_int32_t imesh_stage:4; -/* NDPI_PROTOCOL_HTTP */ + /* NDPI_PROTOCOL_HTTP */ u_int32_t http_setup_dir:2; u_int32_t http_stage:2; u_int32_t http_empty_line_seen:1; u_int32_t http_wait_for_retransmission:1; -/* NDPI_PROTOCOL_GNUTELLA */ + /* NDPI_PROTOCOL_GNUTELLA */ u_int32_t gnutella_stage:2; // 0 - 2 -/* NDPI_CONTENT_MMS */ + /* NDPI_CONTENT_MMS */ u_int32_t mms_stage:2; -/* NDPI_PROTOCOL_YAHOO */ + /* NDPI_PROTOCOL_YAHOO */ u_int32_t yahoo_sip_comm:1; u_int32_t yahoo_http_proxy_stage:2; -/* NDPI_PROTOCOL_MSN */ + /* NDPI_PROTOCOL_MSN */ u_int32_t msn_stage:3; u_int32_t msn_ssl_ft:2; -/* NDPI_PROTOCOL_SSH */ + /* NDPI_PROTOCOL_SSH */ u_int32_t ssh_stage:3; -/* NDPI_PROTOCOL_VNC */ + /* NDPI_PROTOCOL_VNC */ u_int32_t vnc_stage:2; // 0 - 3 -/* NDPI_PROTOCOL_TELNET */ + /* NDPI_PROTOCOL_TELNET */ u_int32_t telnet_stage:2; // 0 - 2 -/* NDPI_PROTOCOL_SSL */ + /* NDPI_PROTOCOL_SSL */ u_int8_t ssl_seen_client_cert:1, ssl_seen_server_cert:1, ssl_stage:2; // 0 - 5 -/* NDPI_PROTOCOL_POSTGRES */ + /* NDPI_PROTOCOL_POSTGRES */ u_int32_t postgres_stage:3; -/* NDPI_PROTOCOL_DIRECT_DOWNLOAD_LINK */ + /* NDPI_PROTOCOL_DIRECT_DOWNLOAD_LINK */ u_int32_t ddlink_server_direction:1; u_int32_t seen_syn:1; u_int32_t seen_syn_ack:1; u_int32_t seen_ack:1; -/* NDPI_PROTOCOL_ICECAST */ + /* NDPI_PROTOCOL_ICECAST */ u_int32_t icecast_stage:1; -/* NDPI_PROTOCOL_DOFUS */ + /* NDPI_PROTOCOL_DOFUS */ u_int32_t dofus_stage:1; -/* NDPI_PROTOCOL_FIESTA */ + /* NDPI_PROTOCOL_FIESTA */ u_int32_t fiesta_stage:2; -/* NDPI_PROTOCOL_WORLDOFWARCRAFT */ + /* NDPI_PROTOCOL_WORLDOFWARCRAFT */ u_int32_t wow_stage:2; -/* NDPI_PROTOCOL_HTTP_APPLICATION_VEOHTV */ + /* NDPI_PROTOCOL_HTTP_APPLICATION_VEOHTV */ u_int32_t veoh_tv_stage:2; -/* NDPI_PROTOCOL_SHOUTCAST */ + /* NDPI_PROTOCOL_SHOUTCAST */ u_int32_t shoutcast_stage:2; -/* NDPI_PROTOCOL_RTP */ + /* NDPI_PROTOCOL_RTP */ u_int32_t rtp_special_packets_seen:1; -/* NDPI_PROTOCOL_MAIL_POP */ + /* NDPI_PROTOCOL_MAIL_POP */ u_int32_t mail_pop_stage:2; -/* NDPI_PROTOCOL_MAIL_IMAP */ + /* NDPI_PROTOCOL_MAIL_IMAP */ u_int32_t mail_imap_stage:3, mail_imap_starttls:2; -/* NDPI_PROTOCOL_SKYPE */ + /* NDPI_PROTOCOL_SKYPE */ u_int8_t skype_packet_id; -/* NDPI_PROTOCOL_CITRIX */ + /* NDPI_PROTOCOL_CITRIX */ u_int8_t citrix_packet_id; -/* NDPI_PROTOCOL_LOTUS_NOTES */ + /* NDPI_PROTOCOL_LOTUS_NOTES */ u_int8_t lotus_notes_packet_id; -/* NDPI_PROTOCOL_TEAMVIEWER */ + /* NDPI_PROTOCOL_TEAMVIEWER */ u_int8_t teamviewer_stage; -/* NDPI_PROTOCOL_ZMQ */ + /* NDPI_PROTOCOL_ZMQ */ u_int8_t prev_zmq_pkt_len; u_char prev_zmq_pkt[10]; -/* NDPI_PROTOCOL_PPSTREAM */ + /* NDPI_PROTOCOL_PPSTREAM */ u_int32_t ppstream_stage:3; -/* NDPI_PROTOCOL_MEMCACHED */ + /* NDPI_PROTOCOL_MEMCACHED */ u_int8_t memcached_matches; -/* NDPI_PROTOCOL_NEST_LOG_SINK */ + /* NDPI_PROTOCOL_NEST_LOG_SINK */ u_int8_t nest_log_sink_matches; } #ifndef WIN32 @@ -608,51 +608,51 @@ struct ndpi_flow_tcp_struct { /* ************************************************** */ struct ndpi_flow_udp_struct { -/* NDPI_PROTOCOL_BATTLEFIELD */ + /* NDPI_PROTOCOL_BATTLEFIELD */ u_int32_t battlefield_msg_id; -/* NDPI_PROTOCOL_SNMP */ + /* NDPI_PROTOCOL_SNMP */ u_int32_t snmp_msg_id; -/* NDPI_PROTOCOL_BATTLEFIELD */ + /* NDPI_PROTOCOL_BATTLEFIELD */ u_int32_t battlefield_stage:3; -/* NDPI_PROTOCOL_SNMP */ + /* NDPI_PROTOCOL_SNMP */ u_int32_t snmp_stage:2; -/* NDPI_PROTOCOL_PPSTREAM */ + /* NDPI_PROTOCOL_PPSTREAM */ u_int32_t ppstream_stage:3; // 0 - 7 -/* NDPI_PROTOCOL_HALFLIFE2 */ + /* NDPI_PROTOCOL_HALFLIFE2 */ u_int32_t halflife2_stage:2; // 0 - 2 -/* NDPI_PROTOCOL_TFTP */ + /* NDPI_PROTOCOL_TFTP */ u_int32_t tftp_stage:1; -/* NDPI_PROTOCOL_AIMINI */ + /* NDPI_PROTOCOL_AIMINI */ u_int32_t aimini_stage:5; -/* NDPI_PROTOCOL_XBOX */ + /* NDPI_PROTOCOL_XBOX */ u_int32_t xbox_stage:1; -/* NDPI_PROTOCOL_WINDOWS_UPDATE */ + /* NDPI_PROTOCOL_WINDOWS_UPDATE */ u_int32_t wsus_stage:1; -/* NDPI_PROTOCOL_SKYPE */ + /* NDPI_PROTOCOL_SKYPE */ u_int8_t skype_packet_id; -/* NDPI_PROTOCOL_TEAMVIEWER */ + /* NDPI_PROTOCOL_TEAMVIEWER */ u_int8_t teamviewer_stage; -/* NDPI_PROTOCOL_EAQ */ + /* NDPI_PROTOCOL_EAQ */ u_int8_t eaq_pkt_id; u_int32_t eaq_sequence; -/* NDPI_PROTOCOL_RX */ + /* NDPI_PROTOCOL_RX */ u_int32_t rx_conn_epoch; u_int32_t rx_conn_id; -/* NDPI_PROTOCOL_MEMCACHED */ + /* NDPI_PROTOCOL_MEMCACHED */ u_int8_t memcached_matches; } #ifndef WIN32 @@ -746,87 +746,87 @@ typedef struct { } ndpi_port_range; typedef enum { - NDPI_PROTOCOL_SAFE = 0, /* Surely doesn't provide risks for the network. (e.g., a news site) */ - NDPI_PROTOCOL_ACCEPTABLE, /* Probably doesn't provide risks, but could be malicious (e.g., Dropbox) */ - NDPI_PROTOCOL_FUN, /* Pure fun protocol, which may be prohibited by the user policy (e.g., Netflix) */ - NDPI_PROTOCOL_UNSAFE, /* Probably provides risks, but could be a normal traffic. Unencrypted protocols with clear pass should be here (e.g., telnet) */ - NDPI_PROTOCOL_POTENTIALLY_DANGEROUS, /* Surely is dangerous (ex. Tor). Be prepared to troubles */ - NDPI_PROTOCOL_TRACKER_ADS, /* Trackers, Advertisements... */ - NDPI_PROTOCOL_UNRATED /* No idea, not implemented or impossible to classify */ + NDPI_PROTOCOL_SAFE = 0, /* Surely doesn't provide risks for the network. (e.g., a news site) */ + NDPI_PROTOCOL_ACCEPTABLE, /* Probably doesn't provide risks, but could be malicious (e.g., Dropbox) */ + NDPI_PROTOCOL_FUN, /* Pure fun protocol, which may be prohibited by the user policy (e.g., Netflix) */ + NDPI_PROTOCOL_UNSAFE, /* Probably provides risks, but could be a normal traffic. Unencrypted protocols with clear pass should be here (e.g., telnet) */ + NDPI_PROTOCOL_POTENTIALLY_DANGEROUS, /* Surely is dangerous (ex. Tor). Be prepared to troubles */ + NDPI_PROTOCOL_TRACKER_ADS, /* Trackers, Advertisements... */ + NDPI_PROTOCOL_UNRATED /* No idea, not implemented or impossible to classify */ } ndpi_protocol_breed_t; #define NUM_BREEDS (NDPI_PROTOCOL_UNRATED+1) /* Abstract categories to group the protocols. */ typedef enum { - NDPI_PROTOCOL_CATEGORY_UNSPECIFIED = 0, /* For general services and unknown protocols */ - NDPI_PROTOCOL_CATEGORY_MEDIA, /* Multimedia and streaming */ - NDPI_PROTOCOL_CATEGORY_VPN, /* Virtual Private Networks */ - NDPI_PROTOCOL_CATEGORY_MAIL, /* Protocols to send/receive/sync emails */ - NDPI_PROTOCOL_CATEGORY_DATA_TRANSFER, /* AFS/NFS and similar protocols */ - NDPI_PROTOCOL_CATEGORY_WEB, /* Web/mobile protocols and services */ - NDPI_PROTOCOL_CATEGORY_SOCIAL_NETWORK, /* Social networks */ - NDPI_PROTOCOL_CATEGORY_DOWNLOAD_FT, /* Download, FTP, file transfer/sharing */ - NDPI_PROTOCOL_CATEGORY_GAME, /* Online games */ - NDPI_PROTOCOL_CATEGORY_CHAT, /* Instant messaging */ - NDPI_PROTOCOL_CATEGORY_VOIP, /* Real-time communications and conferencing */ - NDPI_PROTOCOL_CATEGORY_DATABASE, /* Protocols for database communication */ - NDPI_PROTOCOL_CATEGORY_REMOTE_ACCESS, /* Remote access and control */ - NDPI_PROTOCOL_CATEGORY_CLOUD, /* Online cloud services */ - NDPI_PROTOCOL_CATEGORY_NETWORK, /* Network infrastructure protocols */ - NDPI_PROTOCOL_CATEGORY_COLLABORATIVE, /* Software for collaborative development, including Webmail */ - NDPI_PROTOCOL_CATEGORY_RPC, /* High level network communication protocols */ - NDPI_PROTOCOL_CATEGORY_STREAMING, /* Streaming protocols */ - NDPI_PROTOCOL_CATEGORY_SYSTEM_OS, /* System/Operating System level applications */ - NDPI_PROTOCOL_CATEGORY_SW_UPDATE, /* Software update */ - - /* See #define NUM_CUSTOM_CATEGORIES */ - NDPI_PROTOCOL_CATEGORY_CUSTOM_1, /* User custom category 1 */ - NDPI_PROTOCOL_CATEGORY_CUSTOM_2, /* User custom category 2 */ - NDPI_PROTOCOL_CATEGORY_CUSTOM_3, /* User custom category 3 */ - NDPI_PROTOCOL_CATEGORY_CUSTOM_4, /* User custom category 4 */ - NDPI_PROTOCOL_CATEGORY_CUSTOM_5, /* User custom category 5 */ - - /* Payload Content */ - NDPI_CONTENT_CATEGORY_AVI, - NDPI_CONTENT_CATEGORY_FLASH, - NDPI_CONTENT_CATEGORY_OGG, - NDPI_CONTENT_CATEGORY_MPEG, - NDPI_CONTENT_CATEGORY_QUICKTIME, - NDPI_CONTENT_CATEGORY_REALMEDIA, - NDPI_CONTENT_CATEGORY_WINDOWSMEDIA, - NDPI_CONTENT_CATEGORY_WEBM, - - /* Some custom categories */ - CUSTOM_CATEGORY_MINING = 99, - CUSTOM_CATEGORY_MALWARE = 100, - CUSTOM_CATEGORY_ADVERTISEMENT = 101, - CUSTOM_CATEGORY_BANNED_SITE = 102, - CUSTOM_CATEGORY_SITE_UNAVAILABLE = 103, - - /* - IMPORTANT - - Please keep in sync with - - static const char* categories[] = { ..} - - in ndpi_main.c - */ + NDPI_PROTOCOL_CATEGORY_UNSPECIFIED = 0, /* For general services and unknown protocols */ + NDPI_PROTOCOL_CATEGORY_MEDIA, /* Multimedia and streaming */ + NDPI_PROTOCOL_CATEGORY_VPN, /* Virtual Private Networks */ + NDPI_PROTOCOL_CATEGORY_MAIL, /* Protocols to send/receive/sync emails */ + NDPI_PROTOCOL_CATEGORY_DATA_TRANSFER, /* AFS/NFS and similar protocols */ + NDPI_PROTOCOL_CATEGORY_WEB, /* Web/mobile protocols and services */ + NDPI_PROTOCOL_CATEGORY_SOCIAL_NETWORK, /* Social networks */ + NDPI_PROTOCOL_CATEGORY_DOWNLOAD_FT, /* Download, FTP, file transfer/sharing */ + NDPI_PROTOCOL_CATEGORY_GAME, /* Online games */ + NDPI_PROTOCOL_CATEGORY_CHAT, /* Instant messaging */ + NDPI_PROTOCOL_CATEGORY_VOIP, /* Real-time communications and conferencing */ + NDPI_PROTOCOL_CATEGORY_DATABASE, /* Protocols for database communication */ + NDPI_PROTOCOL_CATEGORY_REMOTE_ACCESS, /* Remote access and control */ + NDPI_PROTOCOL_CATEGORY_CLOUD, /* Online cloud services */ + NDPI_PROTOCOL_CATEGORY_NETWORK, /* Network infrastructure protocols */ + NDPI_PROTOCOL_CATEGORY_COLLABORATIVE, /* Software for collaborative development, including Webmail */ + NDPI_PROTOCOL_CATEGORY_RPC, /* High level network communication protocols */ + NDPI_PROTOCOL_CATEGORY_STREAMING, /* Streaming protocols */ + NDPI_PROTOCOL_CATEGORY_SYSTEM_OS, /* System/Operating System level applications */ + NDPI_PROTOCOL_CATEGORY_SW_UPDATE, /* Software update */ + + /* See #define NUM_CUSTOM_CATEGORIES */ + NDPI_PROTOCOL_CATEGORY_CUSTOM_1, /* User custom category 1 */ + NDPI_PROTOCOL_CATEGORY_CUSTOM_2, /* User custom category 2 */ + NDPI_PROTOCOL_CATEGORY_CUSTOM_3, /* User custom category 3 */ + NDPI_PROTOCOL_CATEGORY_CUSTOM_4, /* User custom category 4 */ + NDPI_PROTOCOL_CATEGORY_CUSTOM_5, /* User custom category 5 */ + + /* Payload Content */ + NDPI_CONTENT_CATEGORY_AVI, + NDPI_CONTENT_CATEGORY_FLASH, + NDPI_CONTENT_CATEGORY_OGG, + NDPI_CONTENT_CATEGORY_MPEG, + NDPI_CONTENT_CATEGORY_QUICKTIME, + NDPI_CONTENT_CATEGORY_REALMEDIA, + NDPI_CONTENT_CATEGORY_WINDOWSMEDIA, + NDPI_CONTENT_CATEGORY_WEBM, + + /* Some custom categories */ + CUSTOM_CATEGORY_MINING = 99, + CUSTOM_CATEGORY_MALWARE = 100, + CUSTOM_CATEGORY_ADVERTISEMENT = 101, + CUSTOM_CATEGORY_BANNED_SITE = 102, + CUSTOM_CATEGORY_SITE_UNAVAILABLE = 103, + + /* + IMPORTANT + + Please keep in sync with + + static const char* categories[] = { ..} + + in ndpi_main.c + */ - NDPI_PROTOCOL_NUM_CATEGORIES /* - NOTE: Keep this as last member - Unused as value but useful to getting the number of elements - in this datastructure - */ + NDPI_PROTOCOL_NUM_CATEGORIES /* + NOTE: Keep this as last member + Unused as value but useful to getting the number of elements + in this datastructure + */ } ndpi_protocol_category_t; typedef enum { - ndpi_pref_http_dont_dissect_response = 0, - ndpi_pref_dns_dissect_response, - ndpi_pref_direction_detect_disable, - ndpi_pref_disable_metadata_export, - ndpi_pref_enable_category_substring_match + ndpi_pref_http_dont_dissect_response = 0, + ndpi_pref_dns_dissect_response, + ndpi_pref_direction_detect_disable, + ndpi_pref_disable_metadata_export, + ndpi_pref_enable_category_substring_match } ndpi_detection_preference; /* ntop extensions */ @@ -875,9 +875,9 @@ typedef struct ndpi_proto { #include <hs/hs.h> struct hs_list { - char *expression; - unsigned int id; - struct hs_list *next; + char *expression; + unsigned int id; + struct hs_list *next; }; struct hs { @@ -986,13 +986,13 @@ struct ndpi_detection_module_struct { char ip_string[NDPI_IP_STRING_SIZE]; #endif u_int8_t ip_version_limit; -/* NDPI_PROTOCOL_BITTORRENT */ + /* NDPI_PROTOCOL_BITTORRENT */ struct hash_ip4p_table *bt_ht; #ifdef NDPI_DETECTION_SUPPORT_IPV6 struct hash_ip4p_table *bt6_ht; #endif -/* BT_ANNOUNCE */ + /* BT_ANNOUNCE */ struct bt_announce *bt_ann; int bt_ann_len; @@ -1122,7 +1122,7 @@ struct ndpi_flow_struct { char fingerprint[48]; char class_ident[48]; } dhcp; - } protos; + } protos; /*** ALL protocol specific 64 bit variables here ***/ @@ -1131,90 +1131,83 @@ struct ndpi_flow_struct { ndpi_protocol_category_t category; -/* NDPI_PROTOCOL_REDIS */ + /* NDPI_PROTOCOL_REDIS */ u_int8_t redis_s2d_first_char, redis_d2s_first_char; u_int16_t packet_counter; // can be 0 - 65000 u_int16_t packet_direction_counter[2]; u_int16_t byte_counter[2]; -/* NDPI_PROTOCOL_BITTORRENT */ + /* NDPI_PROTOCOL_BITTORRENT */ u_int8_t bittorrent_stage; // can be 0 - 255 -/* NDPI_PROTOCOL_DIRECTCONNECT */ - u_int32_t directconnect_stage:2; // 0 - 1 - -/* NDPI_PROTOCOL_YAHOO */ - u_int32_t sip_yahoo_voice:1; + /* NDPI_PROTOCOL_DIRECTCONNECT */ + u_int8_t directconnect_stage:2; // 0 - 1 -/* NDPI_PROTOCOL_HTTP */ - u_int32_t http_detected:1; + /* NDPI_PROTOCOL_YAHOO */ + u_int8_t sip_yahoo_voice:1; -/* NDPI_PROTOCOL_RTSP */ - u_int32_t rtsprdt_stage:2; - u_int32_t rtsp_control_flow:1; + /* NDPI_PROTOCOL_HTTP */ + u_int8_t http_detected:1; + u_int16_t http_upper_protocol, http_lower_protocol; + + /* NDPI_PROTOCOL_RTSP */ + u_int8_t rtsprdt_stage:2, rtsp_control_flow:1; -/* NDPI_PROTOCOL_YAHOO */ - u_int32_t yahoo_detection_finished:2; + /* NDPI_PROTOCOL_YAHOO */ + u_int8_t yahoo_detection_finished:2; -/* NDPI_PROTOCOL_ZATTOO */ - u_int32_t zattoo_stage:3; + /* NDPI_PROTOCOL_ZATTOO */ + u_int8_t zattoo_stage:3; -/* NDPI_PROTOCOL_QQ */ - u_int32_t qq_stage:3; + /* NDPI_PROTOCOL_QQ */ + u_int8_t qq_stage:3; -/* NDPI_PROTOCOL_THUNDER */ - u_int32_t thunder_stage:2; // 0 - 3 + /* NDPI_PROTOCOL_THUNDER */ + u_int8_t thunder_stage:2; // 0 - 3 -/* NDPI_PROTOCOL_OSCAR */ - u_int32_t oscar_ssl_voice_stage:3; - u_int32_t oscar_video_voice:1; + /* NDPI_PROTOCOL_OSCAR */ + u_int8_t oscar_ssl_voice_stage:3, oscar_video_voice:1; -/* NDPI_PROTOCOL_FLORENSIA */ - u_int32_t florensia_stage:1; + /* NDPI_PROTOCOL_FLORENSIA */ + u_int8_t florensia_stage:1; -/* NDPI_PROTOCOL_SOCKS */ - u_int32_t socks5_stage:2; // 0 - 3 - u_int32_t socks4_stage:2; // 0 - 3 + /* NDPI_PROTOCOL_SOCKS */ + u_int8_t socks5_stage:2, socks4_stage:2; // 0 - 3 -/* NDPI_PROTOCOL_EDONKEY */ - u_int32_t edonkey_stage:2; // 0 - 3 + /* NDPI_PROTOCOL_EDONKEY */ + u_int8_t edonkey_stage:2; // 0 - 3 -/* NDPI_PROTOCOL_FTP_CONTROL */ - u_int32_t ftp_control_stage:2; + /* NDPI_PROTOCOL_FTP_CONTROL */ + u_int8_t ftp_control_stage:2; -/* NDPI_PROTOCOL_RTMP */ - u_int32_t rtmp_stage:2; + /* NDPI_PROTOCOL_RTMP */ + u_int8_t rtmp_stage:2; -/* NDPI_PROTOCOL_PANDO */ - u_int32_t pando_stage:3; + /* NDPI_PROTOCOL_PANDO */ + u_int8_t pando_stage:3; -/* NDPI_PROTOCOL_STEAM */ - u_int32_t steam_stage:3; - u_int32_t steam_stage1:3; // 0 - 4 - u_int32_t steam_stage2:2; // 0 - 2 - u_int32_t steam_stage3:2; // 0 - 2 + /* NDPI_PROTOCOL_STEAM */ + u_int16_t steam_stage:3, steam_stage1:3, steam_stage2:2, steam_stage3:2; -/* NDPI_PROTOCOL_PPLIVE */ - u_int32_t pplive_stage1:3; // 0 - 6 - u_int32_t pplive_stage2:2; // 0 - 2 - u_int32_t pplive_stage3:2; // 0 - 2 + /* NDPI_PROTOCOL_PPLIVE */ + u_int8_t pplive_stage1:3, pplive_stage2:2, pplive_stage3:2; -/* NDPI_PROTOCOL_STARCRAFT */ - u_int32_t starcraft_udp_stage : 3; // 0-7 + /* NDPI_PROTOCOL_STARCRAFT */ + u_int8_t starcraft_udp_stage : 3; // 0-7 -/* NDPI_PROTOCOL_OPENVPN */ + /* NDPI_PROTOCOL_OPENVPN */ u_int8_t ovpn_session_id[8]; u_int8_t ovpn_counter; -/* NDPI_PROTOCOL_TINC */ + /* NDPI_PROTOCOL_TINC */ u_int8_t tinc_state; struct tinc_cache_entry tinc_cache_entry; -/* NDPI_PROTOCOL_CSGO */ + /* NDPI_PROTOCOL_CSGO */ u_int8_t csgo_strid[18],csgo_state,csgo_s2; u_int32_t csgo_id2; -/* NDPI_PROTOCOL_1KXUN || NDPI_PROTOCOL_IQIYI */ + /* NDPI_PROTOCOL_1KXUN || NDPI_PROTOCOL_IQIYI */ u_int16_t kxun_counter, iqiyi_counter; /* internal structures to save functions calls */ diff --git a/src/lib/ndpi_main.c b/src/lib/ndpi_main.c index b8d8f358d..863943c4a 100644 --- a/src/lib/ndpi_main.c +++ b/src/lib/ndpi_main.c @@ -3901,8 +3901,7 @@ void check_ndpi_tcp_flow_func(struct ndpi_detection_module_struct *ndpi_struct, if((proto_id != NDPI_PROTOCOL_UNKNOWN) && NDPI_BITMASK_COMPARE(flow->excluded_protocol_bitmask, ndpi_struct->callback_buffer[proto_index].excluded_protocol_bitmask) == 0 - && NDPI_BITMASK_COMPARE(ndpi_struct->callback_buffer[proto_index].detection_bitmask, - detection_bitmask) != 0 + && NDPI_BITMASK_COMPARE(ndpi_struct->callback_buffer[proto_index].detection_bitmask, detection_bitmask) != 0 && (ndpi_struct->callback_buffer[proto_index].ndpi_selection_bitmask & *ndpi_selection_packet) == ndpi_struct->callback_buffer[proto_index].ndpi_selection_bitmask) { if((flow->guessed_protocol_id != NDPI_PROTOCOL_UNKNOWN) && (ndpi_struct->proto_defaults[flow->guessed_protocol_id].func != NULL)) @@ -3920,7 +3919,6 @@ void check_ndpi_tcp_flow_func(struct ndpi_detection_module_struct *ndpi_struct, detection_bitmask) != 0) { ndpi_struct->callback_buffer_tcp_payload[a].func(ndpi_struct, flow); - if(flow->detected_protocol_stack[0] != NDPI_PROTOCOL_UNKNOWN) break; /* Stop after detecting the first protocol */ } @@ -4731,14 +4729,13 @@ void ndpi_parse_packet_line_info(struct ndpi_detection_module_struct *ndpi_struc { u_int32_t a; struct ndpi_packet_struct *packet = &flow->packet; + if(packet->packet_lines_parsed_complete != 0) return; packet->packet_lines_parsed_complete = 1; packet->parsed_lines = 0; - packet->empty_line_position_set = 0; - packet->host_line.ptr = NULL; packet->host_line.len = 0; packet->referer_line.ptr = NULL; @@ -4779,7 +4776,6 @@ void ndpi_parse_packet_line_info(struct ndpi_detection_module_struct *ndpi_struc packet->line[packet->parsed_lines].len = 0; for(a = 0; a < packet->payload_packet_len; a++) { - if((a + 1) == packet->payload_packet_len) return; /* Return if only one byte remains (prevent invalid reads past end-of-buffer) */ @@ -4788,22 +4784,22 @@ void ndpi_parse_packet_line_info(struct ndpi_detection_module_struct *ndpi_struc /* First line of a HTTP response parsing. Expected a "HTTP/1.? ???" */ if(packet->parsed_lines == 0 && packet->line[0].len >= NDPI_STATICSTRING_LEN("HTTP/1.X 200 ") && - strncasecmp((const char *)packet->line[0].ptr, "HTTP/1.", NDPI_STATICSTRING_LEN("HTTP/1.")) == 0 && - packet->line[0].ptr[NDPI_STATICSTRING_LEN("HTTP/1.X ")] > '0' && /* response code between 000 and 699 */ - packet->line[0].ptr[NDPI_STATICSTRING_LEN("HTTP/1.X ")] < '6') { - - packet->http_response.ptr = &packet->line[0].ptr[NDPI_STATICSTRING_LEN("HTTP/1.1 ")]; - packet->http_response.len = packet->line[0].len - NDPI_STATICSTRING_LEN("HTTP/1.1 "); - packet->http_num_headers++; - - /* Set server HTTP response code */ - strncpy((char*)flow->http.response_status_code, (char*)packet->http_response.ptr, 3); - flow->http.response_status_code[4]='\0'; - - NDPI_LOG_DBG2(ndpi_struct, - "ndpi_parse_packet_line_info: HTTP response parsed: \"%.*s\"\n", - packet->http_response.len, packet->http_response.ptr); + strncasecmp((const char *)packet->line[0].ptr, "HTTP/1.", NDPI_STATICSTRING_LEN("HTTP/1.")) == 0 && + packet->line[0].ptr[NDPI_STATICSTRING_LEN("HTTP/1.X ")] > '0' && /* response code between 000 and 699 */ + packet->line[0].ptr[NDPI_STATICSTRING_LEN("HTTP/1.X ")] < '6') { + packet->http_response.ptr = &packet->line[0].ptr[NDPI_STATICSTRING_LEN("HTTP/1.1 ")]; + packet->http_response.len = packet->line[0].len - NDPI_STATICSTRING_LEN("HTTP/1.1 "); + packet->http_num_headers++; + + /* Set server HTTP response code */ + strncpy((char*)flow->http.response_status_code, (char*)packet->http_response.ptr, 3); + flow->http.response_status_code[4] = '\0'; + + NDPI_LOG_DBG2(ndpi_struct, + "ndpi_parse_packet_line_info: HTTP response parsed: \"%.*s\"\n", + packet->http_response.len, packet->http_response.ptr); } + /* "Server:" header line in HTTP response */ if(packet->line[packet->parsed_lines].len > NDPI_STATICSTRING_LEN("Server:") + 1 && strncasecmp((const char *)packet->line[packet->parsed_lines].ptr, "Server:", NDPI_STATICSTRING_LEN("Server:")) == 0) { @@ -5253,9 +5249,8 @@ void ndpi_int_reset_protocol(struct ndpi_flow_struct *flow) { if(flow) { int a; - for(a = 0; a < NDPI_PROTOCOL_SIZE; a++) { - flow->detected_protocol_stack[a] = NDPI_PROTOCOL_UNKNOWN; - } + for(a = 0; a < NDPI_PROTOCOL_SIZE; a++) + flow->detected_protocol_stack[a] = NDPI_PROTOCOL_UNKNOWN; } } diff --git a/src/lib/protocols/apple_push.c b/src/lib/protocols/apple_push.c index 6930dba86..45346e07b 100644 --- a/src/lib/protocols/apple_push.c +++ b/src/lib/protocols/apple_push.c @@ -31,20 +31,22 @@ static void ndpi_check_apple_push(struct ndpi_detection_module_struct *ndpi_stru struct ndpi_flow_struct *flow) { struct ndpi_packet_struct *packet = &flow->packet; - /* https://support.apple.com/en-us/HT203609 */ - if(((ntohl(packet->iph->saddr) & 0xFF000000 /* 255.0.0.0 */) == 0x11000000 /* 17.0.0.0/8 */) - || ((ntohl(packet->iph->daddr) & 0xFF000000 /* 255.0.0.0 */) == 0x11000000 /* 17.0.0.0/8 */)) { - u_int16_t apple_push_port = ntohs(5223); - u_int16_t notification_apn_port = ntohs(2195); - u_int16_t apn_feedback_port = ntohs(2196); - - if(((packet->tcp->source == apple_push_port) || (packet->tcp->dest == apple_push_port)) - || ((packet->tcp->source == notification_apn_port) || (packet->tcp->dest == notification_apn_port)) - || ((packet->tcp->source == apn_feedback_port) || (packet->tcp->dest == apn_feedback_port)) - ) { - NDPI_LOG_INFO(ndpi_struct, "found apple_push\n"); - ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_APPLE_PUSH, NDPI_PROTOCOL_UNKNOWN); - return; + if(packet->iph) { + /* https://support.apple.com/en-us/HT203609 */ + if(((ntohl(packet->iph->saddr) & 0xFF000000 /* 255.0.0.0 */) == 0x11000000 /* 17.0.0.0/8 */) + || ((ntohl(packet->iph->daddr) & 0xFF000000 /* 255.0.0.0 */) == 0x11000000 /* 17.0.0.0/8 */)) { + u_int16_t apple_push_port = ntohs(5223); + u_int16_t notification_apn_port = ntohs(2195); + u_int16_t apn_feedback_port = ntohs(2196); + + if(((packet->tcp->source == apple_push_port) || (packet->tcp->dest == apple_push_port)) + || ((packet->tcp->source == notification_apn_port) || (packet->tcp->dest == notification_apn_port)) + || ((packet->tcp->source == apn_feedback_port) || (packet->tcp->dest == apn_feedback_port)) + ) { + NDPI_LOG_INFO(ndpi_struct, "found apple_push\n"); + ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_APPLE_PUSH, NDPI_PROTOCOL_UNKNOWN); + return; + } } } diff --git a/src/lib/protocols/http.c b/src/lib/protocols/http.c index 09b816129..e92701072 100644 --- a/src/lib/protocols/http.c +++ b/src/lib/protocols/http.c @@ -28,27 +28,41 @@ #include "ndpi_api.h" #include "lruc.h" -/* global variables used for 1kxun protocol and iqiyi service */ - static void ndpi_int_http_add_connection(struct ndpi_detection_module_struct *ndpi_struct, struct ndpi_flow_struct *flow, u_int16_t category) { - +#ifdef DEBUG + printf("[%s] [http_dont_dissect_response: %u]->> %s\n", __FUNCTION__, + ndpi_struct->http_dont_dissect_response, flow->http.response_status_code); +#endif + if(flow->detected_protocol_stack[0] == NDPI_PROTOCOL_UNKNOWN) { /* This is HTTP and it is not a sub protocol (e.g. skype or dropbox) */ ndpi_search_tcp_or_udp(ndpi_struct, flow); /* If no custom protocol has been detected */ - if(flow->guessed_host_protocol_id != NDPI_PROTOCOL_UNKNOWN) { ndpi_int_reset_protocol(flow); - ndpi_set_detected_protocol(ndpi_struct, flow, flow->guessed_host_protocol_id, NDPI_PROTOCOL_HTTP); + flow->http_upper_protocol = flow->guessed_host_protocol_id, flow->http_lower_protocol = NDPI_PROTOCOL_HTTP; } else - ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_HTTP, NDPI_PROTOCOL_UNKNOWN); - - flow->http_detected = 1, flow->guessed_category = category; + flow->http_upper_protocol = NDPI_PROTOCOL_HTTP, flow->http_lower_protocol = NDPI_PROTOCOL_UNKNOWN; + + if(ndpi_struct->http_dont_dissect_response) + ndpi_set_detected_protocol(ndpi_struct, flow, flow->http_upper_protocol, flow->http_lower_protocol); + else { + flow->detected_protocol_stack[0] = NDPI_PROTOCOL_UNKNOWN, flow->detected_protocol_stack[1] = NDPI_PROTOCOL_UNKNOWN; + flow->packet.detected_protocol_stack[0] = NDPI_PROTOCOL_UNKNOWN, flow->packet.detected_protocol_stack[1] = NDPI_PROTOCOL_UNKNOWN; + } + } else { + if((!ndpi_struct->http_dont_dissect_response) && (flow->http.response_status_code[0] == '\0')) { + flow->http_upper_protocol = flow->detected_protocol_stack[0], flow->http_lower_protocol = flow->detected_protocol_stack[1]; + flow->detected_protocol_stack[0] = NDPI_PROTOCOL_UNKNOWN, flow->detected_protocol_stack[1] = NDPI_PROTOCOL_UNKNOWN; + flow->packet.detected_protocol_stack[0] = NDPI_PROTOCOL_UNKNOWN, flow->packet.detected_protocol_stack[1] = NDPI_PROTOCOL_UNKNOWN; + } } + + flow->http_detected = 1, flow->guessed_category = category; } #ifdef NDPI_CONTENT_CATEGORY_FLASH @@ -79,7 +93,7 @@ static void avi_check_http_payload(struct ndpi_detection_module_struct *ndpi_str NDPI_LOG_DBG2(ndpi_struct, "called avi_check_http_payload: %u %u %u\n", - packet->empty_line_position_set, flow->l4.tcp.http_empty_line_seen, packet->empty_line_position); + packet->empty_line_position_set, flow->l4.tcp.http_empty_line_seen, packet->empty_line_position); if(packet->empty_line_position_set == 0 && flow->l4.tcp.http_empty_line_seen == 0) return; @@ -125,7 +139,7 @@ static void teamviewer_check_http_payload(struct ndpi_detection_module_struct *n const u_int8_t *pos; NDPI_LOG_DBG2(ndpi_struct, "called teamviewer_check_http_payload: %u %u %u\n", - packet->empty_line_position_set, flow->l4.tcp.http_empty_line_seen, packet->empty_line_position); + packet->empty_line_position_set, flow->l4.tcp.http_empty_line_seen, packet->empty_line_position); if(packet->empty_line_position_set == 0 || (packet->empty_line_position + 5) > (packet->payload_packet_len)) return; @@ -171,10 +185,10 @@ static void setHttpUserAgent(struct ndpi_detection_module_struct *ndpi_struct, static void parseHttpSubprotocol(struct ndpi_detection_module_struct *ndpi_struct, struct ndpi_flow_struct *flow) { if((flow->l4.tcp.http_stage == 0) || (flow->http.url && flow->http_detected)) { - char *double_col = strchr((char*)flow->host_server_name, ':'); - ndpi_protocol_match_result ret_match; - - if(double_col) double_col[0] = '\0'; + char *double_col = strchr((char*)flow->host_server_name, ':'); + ndpi_protocol_match_result ret_match; + + if(double_col) double_col[0] = '\0'; /** NOTE @@ -198,6 +212,13 @@ static void check_content_type_and_change_protocol(struct ndpi_detection_module_ struct ndpi_packet_struct *packet = &flow->packet; u_int8_t a; + if((!ndpi_struct->http_dont_dissect_response) && flow->http_detected && (flow->http.response_status_code[0] != 0)) { + ndpi_set_detected_protocol(ndpi_struct, flow, flow->http_upper_protocol, flow->http_lower_protocol); +#ifdef DEBUG + printf("[%s] [http_dont_dissect_response: %u]->> %s\n", __FUNCTION__, ndpi_struct->http_dont_dissect_response, flow->http.response_status_code); +#endif + return; + } #if defined(NDPI_PROTOCOL_1KXUN) || defined(NDPI_PROTOCOL_IQIYI) /* PPStream */ @@ -277,8 +298,8 @@ static void check_content_type_and_change_protocol(struct ndpi_detection_module_ if(packet->user_agent_line.ptr != NULL && packet->user_agent_line.len != 0) { /** Format examples: - Mozilla/5.0 (iPad; U; CPU OS 3_2 like Mac OS X; en-us) AppleWebKit/531.21.10 (KHTML, like Gecko) .... - Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:54.0) Gecko/20100101 Firefox/54.0 + Mozilla/5.0 (iPad; U; CPU OS 3_2 like Mac OS X; en-us) AppleWebKit/531.21.10 (KHTML, like Gecko) .... + Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:54.0) Gecko/20100101 Firefox/54.0 */ if(packet->user_agent_line.len > 7) { char ua[256]; @@ -308,22 +329,22 @@ static void check_content_type_and_change_protocol(struct ndpi_detection_module_ if(token && (token[0] == ' ')) token++; /* Skip space */ if(token - && ((strcmp(token, "U") == 0) - || (strncmp(token, "MSIE", 4) == 0))) { - token = strsep(&parent, ";"); - if(token && (token[0] == ' ')) token++; /* Skip space */ + && ((strcmp(token, "U") == 0) + || (strncmp(token, "MSIE", 4) == 0))) { + token = strsep(&parent, ";"); + if(token && (token[0] == ' ')) token++; /* Skip space */ - if(token && (strncmp(token, "Update", 6) == 0)) { - token = strsep(&parent, ";"); + if(token && (strncmp(token, "Update", 6) == 0)) { + token = strsep(&parent, ";"); - if(token && (token[0] == ' ')) token++; /* Skip space */ + if(token && (token[0] == ' ')) token++; /* Skip space */ - if(token && (strncmp(token, "AOL", 3) == 0)) { + if(token && (strncmp(token, "AOL", 3) == 0)) { - token = strsep(&parent, ";"); - if(token && (token[0] == ' ')) token++; /* Skip space */ - } - } + token = strsep(&parent, ";"); + if(token && (token[0] == ' ')) token++; /* Skip space */ + } + } } } @@ -340,7 +361,7 @@ static void check_content_type_and_change_protocol(struct ndpi_detection_module_ } NDPI_LOG_DBG2(ndpi_struct, "User Agent Type line found %.*s\n", - packet->user_agent_line.len, packet->user_agent_line.ptr); + packet->user_agent_line.len, packet->user_agent_line.ptr); } /* check for host line */ @@ -348,37 +369,37 @@ static void check_content_type_and_change_protocol(struct ndpi_detection_module_ u_int len; NDPI_LOG_DBG2(ndpi_struct, "HOST line found %.*s\n", - packet->host_line.len, packet->host_line.ptr); + packet->host_line.len, packet->host_line.ptr); /* call ndpi_match_host_subprotocol to see if there is a match with known-host HTTP subprotocol */ if((ndpi_struct->http_dont_dissect_response) || flow->http_detected) { ndpi_protocol_match_result ret_match; - + ndpi_match_host_subprotocol(ndpi_struct, flow, (char*)packet->host_line.ptr, packet->host_line.len, &ret_match, NDPI_PROTOCOL_HTTP); } - + /* Copy result for nDPI apps */ if(!ndpi_struct->disable_metadata_export) { len = ndpi_min(packet->host_line.len, sizeof(flow->host_server_name)-1); strncpy((char*)flow->host_server_name, (char*)packet->host_line.ptr, len); flow->host_server_name[len] = '\0'; } - + flow->server_id = flow->dst; if(packet->forwarded_line.ptr) { - len = ndpi_min(packet->forwarded_line.len, sizeof(flow->protos.http.nat_ip)-1); - if(!ndpi_struct->disable_metadata_export) { - strncpy((char*)flow->protos.http.nat_ip, (char*)packet->forwarded_line.ptr, len); - flow->protos.http.nat_ip[len] = '\0'; - } + len = ndpi_min(packet->forwarded_line.len, sizeof(flow->protos.http.nat_ip)-1); + if(!ndpi_struct->disable_metadata_export) { + strncpy((char*)flow->protos.http.nat_ip, (char*)packet->forwarded_line.ptr, len); + flow->protos.http.nat_ip[len] = '\0'; + } } - if(ndpi_struct->http_dont_dissect_response) + if(!ndpi_struct->http_dont_dissect_response) parseHttpSubprotocol(ndpi_struct, flow); /** @@ -403,26 +424,28 @@ static void check_content_type_and_change_protocol(struct ndpi_detection_module_ && ((ndpi_struct->http_dont_dissect_response) || flow->http_detected) && (packet->http_origin.len > 0)) { ndpi_protocol_match_result ret_match; - + ndpi_match_host_subprotocol(ndpi_struct, flow, (char *)packet->http_origin.ptr, packet->http_origin.len, &ret_match, NDPI_PROTOCOL_HTTP); } - + if(flow->detected_protocol_stack[0] != NDPI_PROTOCOL_UNKNOWN) { if(packet->detected_protocol_stack[0] != NDPI_PROTOCOL_HTTP) { - NDPI_LOG_INFO(ndpi_struct, "found HTTP/%s\n", - ndpi_get_proto_name(ndpi_struct, packet->detected_protocol_stack[0])); + NDPI_LOG_INFO(ndpi_struct, "found HTTP/%s\n", + ndpi_get_proto_name(ndpi_struct, packet->detected_protocol_stack[0])); ndpi_int_http_add_connection(ndpi_struct, flow, packet->detected_protocol_stack[0]); return; /* We have identified a sub-protocol so we're done */ } } } +#if 0 if(!ndpi_struct->http_dont_dissect_response && flow->http_detected) parseHttpSubprotocol(ndpi_struct, flow); +#endif if(flow->guessed_protocol_id == NDPI_PROTOCOL_UNKNOWN) flow->guessed_protocol_id = NDPI_PROTOCOL_HTTP; @@ -430,7 +453,7 @@ static void check_content_type_and_change_protocol(struct ndpi_detection_module_ /* check for accept line */ if(packet->accept_line.ptr != NULL) { NDPI_LOG_DBG2(ndpi_struct, "Accept line found %.*s\n", - packet->accept_line.len, packet->accept_line.ptr); + packet->accept_line.len, packet->accept_line.ptr); if(NDPI_COMPARE_PROTOCOL_TO_BITMASK(ndpi_struct->detection_bitmask, NDPI_PROTOCOL_RTSP) != 0) { rtsp_parse_packet_acceptline(ndpi_struct, flow); @@ -438,26 +461,28 @@ static void check_content_type_and_change_protocol(struct ndpi_detection_module_ } /* search for line startin with "Icy-MetaData" */ - for (a = 0; a < packet->parsed_lines; a++) { + for (a = 0; a < packet->parsed_lines; a++) { if(packet->line[a].len > 11 && memcmp(packet->line[a].ptr, "Icy-MetaData", 12) == 0) { - NDPI_LOG_INFO(ndpi_struct, "found MPEG: Icy-MetaData\n"); - ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_CONTENT_CATEGORY_MPEG); - return; - } + NDPI_LOG_INFO(ndpi_struct, "found MPEG: Icy-MetaData\n"); + ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_CONTENT_CATEGORY_MPEG); + return; + } } if(packet->content_line.ptr != NULL && packet->content_line.len != 0) { NDPI_LOG_DBG2(ndpi_struct, "Content Type line found %.*s\n", - packet->content_line.len, packet->content_line.ptr); + packet->content_line.len, packet->content_line.ptr); if((ndpi_struct->http_dont_dissect_response) || flow->http_detected) { ndpi_protocol_match_result ret_match; - + ndpi_match_content_subprotocol(ndpi_struct, flow, (char*)packet->content_line.ptr, packet->content_line.len, &ret_match, NDPI_PROTOCOL_HTTP); } } + + ndpi_int_http_add_connection(ndpi_struct, flow, packet->detected_protocol_stack[0]); } static void check_http_payload(struct ndpi_detection_module_struct *ndpi_struct, struct ndpi_flow_struct *flow) @@ -486,22 +511,22 @@ static void check_http_payload(struct ndpi_detection_module_struct *ndpi_struct, #define STATIC_STRING_L(a) {.str=a, .len=sizeof(a)-1 } static struct l_string { - const char *str; - size_t len; + const char *str; + size_t len; } http_methods[] = { - STATIC_STRING_L("GET "), - STATIC_STRING_L("POST "), - STATIC_STRING_L("OPTIONS "), - STATIC_STRING_L("HEAD "), - STATIC_STRING_L("PUT "), - STATIC_STRING_L("DELETE "), - STATIC_STRING_L("CONNECT "), - STATIC_STRING_L("PROPFIND "), - STATIC_STRING_L("REPORT ") }; + STATIC_STRING_L("GET "), + STATIC_STRING_L("POST "), + STATIC_STRING_L("OPTIONS "), + STATIC_STRING_L("HEAD "), + STATIC_STRING_L("PUT "), + STATIC_STRING_L("DELETE "), + STATIC_STRING_L("CONNECT "), + STATIC_STRING_L("PROPFIND "), + STATIC_STRING_L("REPORT ") }; static const char *http_fs = "CDGHOPR"; static uint8_t non_ctrl(uint8_t c) { - return c < 32 ? '.':c; + return c < 32 ? '.':c; } static u_int16_t http_request_url_offset(struct ndpi_detection_module_struct *ndpi_struct, struct ndpi_flow_struct *flow) @@ -510,9 +535,9 @@ static u_int16_t http_request_url_offset(struct ndpi_detection_module_struct *nd int i; NDPI_LOG_DBG2(ndpi_struct, "====>>>> HTTP: %c%c%c%c [len: %u]\n", - non_ctrl(packet->payload[0]), non_ctrl(packet->payload[1]), - non_ctrl(packet->payload[2]), non_ctrl(packet->payload[3]), - packet->payload_packet_len); + non_ctrl(packet->payload[0]), non_ctrl(packet->payload[1]), + non_ctrl(packet->payload[2]), non_ctrl(packet->payload[3]), + packet->payload_packet_len); /* Check first char */ if(!strchr(http_fs,packet->payload[0])) return 0; @@ -520,11 +545,11 @@ static u_int16_t http_request_url_offset(struct ndpi_detection_module_struct *nd FIRST PAYLOAD PACKET FROM CLIENT **/ for(i=0; i < sizeof(http_methods)/sizeof(http_methods[0]); i++) { - if(packet->payload_packet_len >= http_methods[i].len && - memcmp(packet->payload,http_methods[i].str,http_methods[i].len) == 0) { - NDPI_LOG_DBG2(ndpi_struct, "HTTP: %sFOUND\n",http_methods[i].str); - return http_methods[i].len; - } + if(packet->payload_packet_len >= http_methods[i].len && + memcmp(packet->payload,http_methods[i].str,http_methods[i].len) == 0) { + NDPI_LOG_DBG2(ndpi_struct, "HTTP: %sFOUND\n",http_methods[i].str); + return http_methods[i].len; + } } return 0; } @@ -556,7 +581,7 @@ static void http_bitmask_exclude_other(struct ndpi_flow_struct *flow) /*************************************************************************************************/ static void ndpi_check_http_tcp(struct ndpi_detection_module_struct *ndpi_struct, - struct ndpi_flow_struct *flow) { + struct ndpi_flow_struct *flow) { struct ndpi_packet_struct *packet = &flow->packet; u_int16_t filename_start; /* the filename in the request method line, e.g., "GET filename_start..."*/ @@ -586,7 +611,7 @@ static void ndpi_check_http_tcp(struct ndpi_detection_module_struct *ndpi_struct flow->l4.tcp.http_stage = 1; return; } - + if((packet->payload_packet_len == 40) && (flow->l4.tcp.http_stage == 0)) { /* -> QR O06L0072-6L91-4O43-857J-K8OO172L6L51 @@ -600,11 +625,11 @@ static void ndpi_check_http_tcp(struct ndpi_detection_module_struct *ndpi_struct && (packet->payload[21] == '-') && (packet->payload[26] == '-') && (packet->payload[39] == 0x0A) - ) - flow->l4.tcp.http_stage = 1; - return; + ) + flow->l4.tcp.http_stage = 1; + return; } - + if((packet->payload_packet_len == 23) && (memcmp(packet->payload, "<policy-file-request/>", 23) == 0)) { /* <policy-file-request/> @@ -617,8 +642,8 @@ static void ndpi_check_http_tcp(struct ndpi_detection_module_struct *ndpi_struct ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_OOKLA, NDPI_PROTOCOL_UNKNOWN); if(ndpi_struct->ookla_cache == NULL) - ndpi_struct->ookla_cache = lruc_new(4*1024, 1024); - + ndpi_struct->ookla_cache = lruc_new(4*1024, 1024); + if(ndpi_struct->ookla_cache != NULL) { u_int8_t *dummy = (u_int8_t*)ndpi_malloc(sizeof(u_int8_t)); @@ -629,23 +654,23 @@ static void ndpi_check_http_tcp(struct ndpi_detection_module_struct *ndpi_struct lruc_set((lruc*)ndpi_struct->ookla_cache, (void*)&packet->iph->daddr, 4, dummy, 1); } } - + return; } - + NDPI_EXCLUDE_PROTO(ndpi_struct, flow); http_bitmask_exclude_other(flow); return; } NDPI_LOG_DBG2(ndpi_struct, - "Filename HTTP found: %d, we look for line info..\n", filename_start); + "Filename HTTP found: %d, we look for line info..\n", filename_start); ndpi_parse_packet_line_info(ndpi_struct, flow); if(packet->parsed_lines <= 1) { NDPI_LOG_DBG2(ndpi_struct, - "Found just one line, we will look further for the next packet...\n"); + "Found just one line, we will look further for the next packet...\n"); packet->http_method.ptr = packet->line[0].ptr; packet->http_method.len = filename_start - 1; @@ -656,10 +681,10 @@ static void ndpi_check_http_tcp(struct ndpi_detection_module_struct *ndpi_struct } NDPI_LOG_DBG2(ndpi_struct, - "Found more than one line, we look further for the next packet...\n"); + "Found more than one line, we look further for the next packet...\n"); if(packet->line[0].len >= (9 + filename_start) - && memcmp(&packet->line[0].ptr[packet->line[0].len - 9], " HTTP/1.", 8) == 0) { /* Request line complete. Ex. "GET / HTTP/1.1" */ + && memcmp(&packet->line[0].ptr[packet->line[0].len - 9], " HTTP/1.", 8) == 0) { /* Request line complete. Ex. "GET / HTTP/1.1" */ packet->http_url_name.ptr = &packet->payload[filename_start]; packet->http_url_name.len = packet->line[0].len - (filename_start + 9); @@ -669,14 +694,13 @@ static void ndpi_check_http_tcp(struct ndpi_detection_module_struct *ndpi_struct // Set the HTTP requested version: 0=HTTP/1.0 and 1=HTTP/1.1 if(memcmp(&packet->line[0].ptr[packet->line[0].len - 1], "1", 1) == 0) - flow->http.request_version = 1; + flow->http.request_version = 1; else - flow->http.request_version = 0; + flow->http.request_version = 0; /* Set the first found headers in request */ flow->http.num_request_headers = packet->http_num_headers; - /* Check for Ookla */ if((packet->referer_line.len > 0) && ndpi_strnstr((const char *)packet->referer_line.ptr, "www.speedtest.net", packet->referer_line.len)) { @@ -686,22 +710,22 @@ static void ndpi_check_http_tcp(struct ndpi_detection_module_struct *ndpi_struct /* Check for additional field introduced by Steam */ int x = 1; if(packet->line[x].len >= 11 && (memcmp(packet->line[x].ptr, "x-steam-sid", 11)) == 0) { - NDPI_LOG_INFO(ndpi_struct, "found STEAM\n"); - ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_STEAM); - check_content_type_and_change_protocol(ndpi_struct, flow); - return; + NDPI_LOG_INFO(ndpi_struct, "found STEAM\n"); + ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_STEAM); + check_content_type_and_change_protocol(ndpi_struct, flow); + return; } /* Check for additional field introduced by Facebook */ x = 1; while(packet->line[x].len != 0) { - if(packet->line[x].len >= 12 && (memcmp(packet->line[x].ptr, "X-FB-SIM-HNI", 12)) == 0) { - NDPI_LOG_INFO(ndpi_struct, "found FACEBOOK\n"); - ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_FACEBOOK); - check_content_type_and_change_protocol(ndpi_struct, flow); - return; - } - x++; + if(packet->line[x].len >= 12 && (memcmp(packet->line[x].ptr, "X-FB-SIM-HNI", 12)) == 0) { + NDPI_LOG_INFO(ndpi_struct, "found FACEBOOK\n"); + ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_FACEBOOK); + check_content_type_and_change_protocol(ndpi_struct, flow); + return; + } + x++; } #if defined(NDPI_PROTOCOL_1KXUN) || defined(NDPI_PROTOCOL_IQIYI) @@ -746,23 +770,23 @@ static void ndpi_check_http_tcp(struct ndpi_detection_module_struct *ndpi_struct } } #endif - + if((packet->http_url_name.len > 7) - && (!strncmp((const char*) packet->http_url_name.ptr, "http://", 7))) { + && (!strncmp((const char*) packet->http_url_name.ptr, "http://", 7))) { NDPI_LOG_INFO(ndpi_struct, "found HTTP_PROXY\n"); ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_HTTP_PROXY); check_content_type_and_change_protocol(ndpi_struct, flow); } if(filename_start == 8 && (memcmp(packet->payload, "CONNECT ", 8) == 0)) { - /* nathan@getoffmalawn.com */ + /* nathan@getoffmalawn.com */ NDPI_LOG_INFO(ndpi_struct, "found HTTP_CONNECT\n"); ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_HTTP_CONNECT); check_content_type_and_change_protocol(ndpi_struct, flow); } NDPI_LOG_DBG2(ndpi_struct, - "HTTP START Found, we will look for sub-protocols (content and host)...\n"); + "HTTP START Found, we will look for sub-protocols (content and host)...\n"); if(packet->host_line.ptr != NULL) { /** @@ -775,11 +799,11 @@ static void ndpi_check_http_tcp(struct ndpi_detection_module_struct *ndpi_struct if(ndpi_struct->http_dont_dissect_response) { if(flow->detected_protocol_stack[0] == NDPI_PROTOCOL_UNKNOWN) /* No subprotocol found */ NDPI_LOG_INFO(ndpi_struct, "found HTTP\n"); - ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_HTTP); + ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_HTTP); } else { flow->http_detected = 1; NDPI_LOG_DBG2(ndpi_struct, - "HTTP START Found, we will look further for the response...\n"); + "HTTP START Found, we will look further for the response...\n"); flow->l4.tcp.http_stage = packet->packet_direction + 1; // packet_direction 0: stage 1, packet_direction 1: stage 2 } @@ -790,23 +814,21 @@ static void ndpi_check_http_tcp(struct ndpi_detection_module_struct *ndpi_struct NDPI_EXCLUDE_PROTO(ndpi_struct, flow); http_bitmask_exclude_other(flow); - } else if((flow->l4.tcp.http_stage == 1) || (flow->l4.tcp.http_stage == 2)) { - NDPI_LOG_DBG2(ndpi_struct, "HTTP stage %u: \n", flow->l4.tcp.http_stage); - + if((packet->payload_packet_len == 34) && (flow->l4.tcp.http_stage == 1)) { if((packet->payload[5] == ' ') && (packet->payload[9] == ' ')) { goto ookla_found; } } - + if((packet->payload_packet_len > 6) && memcmp(packet->payload, "HELLO ", 6) == 0) { - /* This looks like Ookla */ + /* This looks like Ookla */ goto ookla_found; } else - NDPI_ADD_PROTOCOL_TO_BITMASK(flow->excluded_protocol_bitmask, NDPI_PROTOCOL_OOKLA); - + NDPI_ADD_PROTOCOL_TO_BITMASK(flow->excluded_protocol_bitmask, NDPI_PROTOCOL_OOKLA); + /** At first check, if this is for sure a response packet (in another direction. If not, if HTTP is detected do nothing now and return, otherwise check the second packet for the HTTP request @@ -817,7 +839,7 @@ static void ndpi_check_http_tcp(struct ndpi_detection_module_struct *ndpi_struct return; NDPI_LOG_DBG2(ndpi_struct, - " SECOND PAYLOAD TRAFFIC FROM CLIENT, FIRST PACKET MIGHT HAVE BEEN HTTP...UNKNOWN TRAFFIC, HERE FOR HTTP again.. \n"); + " SECOND PAYLOAD TRAFFIC FROM CLIENT, FIRST PACKET MIGHT HAVE BEEN HTTP...UNKNOWN TRAFFIC, HERE FOR HTTP again.. \n"); ndpi_parse_packet_line_info(ndpi_struct, flow); @@ -839,14 +861,14 @@ static void ndpi_check_http_tcp(struct ndpi_detection_module_struct *ndpi_struct } // http://www.slideshare.net/DSPIP/rtsp-analysis-wireshark if(packet->line[0].len >= 9 - && memcmp(&packet->line[0].ptr[packet->line[0].len - 9], " HTTP/1.", 8) == 0) { + && memcmp(&packet->line[0].ptr[packet->line[0].len - 9], " HTTP/1.", 8) == 0) { NDPI_LOG_INFO(ndpi_struct, "found HTTP\n"); ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_HTTP); check_content_type_and_change_protocol(ndpi_struct, flow); NDPI_LOG_DBG2(ndpi_struct, - "HTTP START Found in 2. packet, we will look further for the response....\n"); + "HTTP START Found in 2. packet, we will look further for the response....\n"); flow->http_detected = 1; } @@ -879,9 +901,8 @@ static void ndpi_check_http_tcp(struct ndpi_detection_module_struct *ndpi_struct ndpi_parse_packet_line_info(ndpi_struct, flow); check_content_type_and_change_protocol(ndpi_struct, flow); - if(packet->packet_direction == 1 /* server -> client */){ - flow->http.num_response_headers += packet->http_num_headers; /* flow structs are initialized with zeros */ - } + if(packet->packet_direction == 1 /* server -> client */) + flow->http.num_response_headers += packet->http_num_headers; /* flow structs are initialized with zeros */ if(packet->empty_line_position_set != 0 || flow->l4.tcp.http_empty_line_seen == 1) { NDPI_LOG_DBG2(ndpi_struct, "empty line. check_http_payload\n"); @@ -905,8 +926,8 @@ void ndpi_search_http_tcp(struct ndpi_detection_module_struct *ndpi_struct, } if(packet->detected_protocol_stack[0] != NDPI_PROTOCOL_UNKNOWN) { - return; - } + return; + } NDPI_LOG_DBG(ndpi_struct, "search HTTP\n"); ndpi_check_http_tcp(ndpi_struct, flow); @@ -935,7 +956,7 @@ char* ndpi_get_http_url(struct ndpi_detection_module_struct *ndpi_mod, /* ********************************* */ char* ndpi_get_http_content_type(struct ndpi_detection_module_struct *ndpi_mod, - struct ndpi_flow_struct *flow) { + struct ndpi_flow_struct *flow) { if((!flow) || (!flow->http.content_type)) return(""); else |