From 97bdfe295d3f7318d1eac3e0020b3b13004f008d Mon Sep 17 00:00:00 2001 From: Luca Deri Date: Mon, 7 Jan 2019 00:28:29 +0100 Subject: nDPi now finally honours dissection of HTTP responses --- src/lib/protocols/http.c | 263 +++++++++++++++++++++++++---------------------- 1 file changed, 142 insertions(+), 121 deletions(-) (limited to 'src/lib/protocols/http.c') diff --git a/src/lib/protocols/http.c b/src/lib/protocols/http.c index 09b816129..e92701072 100644 --- a/src/lib/protocols/http.c +++ b/src/lib/protocols/http.c @@ -28,27 +28,41 @@ #include "ndpi_api.h" #include "lruc.h" -/* global variables used for 1kxun protocol and iqiyi service */ - static void ndpi_int_http_add_connection(struct ndpi_detection_module_struct *ndpi_struct, struct ndpi_flow_struct *flow, u_int16_t category) { - +#ifdef DEBUG + printf("[%s] [http_dont_dissect_response: %u]->> %s\n", __FUNCTION__, + ndpi_struct->http_dont_dissect_response, flow->http.response_status_code); +#endif + if(flow->detected_protocol_stack[0] == NDPI_PROTOCOL_UNKNOWN) { /* This is HTTP and it is not a sub protocol (e.g. skype or dropbox) */ ndpi_search_tcp_or_udp(ndpi_struct, flow); /* If no custom protocol has been detected */ - if(flow->guessed_host_protocol_id != NDPI_PROTOCOL_UNKNOWN) { ndpi_int_reset_protocol(flow); - ndpi_set_detected_protocol(ndpi_struct, flow, flow->guessed_host_protocol_id, NDPI_PROTOCOL_HTTP); + flow->http_upper_protocol = flow->guessed_host_protocol_id, flow->http_lower_protocol = NDPI_PROTOCOL_HTTP; } else - ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_HTTP, NDPI_PROTOCOL_UNKNOWN); - - flow->http_detected = 1, flow->guessed_category = category; + flow->http_upper_protocol = NDPI_PROTOCOL_HTTP, flow->http_lower_protocol = NDPI_PROTOCOL_UNKNOWN; + + if(ndpi_struct->http_dont_dissect_response) + ndpi_set_detected_protocol(ndpi_struct, flow, flow->http_upper_protocol, flow->http_lower_protocol); + else { + flow->detected_protocol_stack[0] = NDPI_PROTOCOL_UNKNOWN, flow->detected_protocol_stack[1] = NDPI_PROTOCOL_UNKNOWN; + flow->packet.detected_protocol_stack[0] = NDPI_PROTOCOL_UNKNOWN, flow->packet.detected_protocol_stack[1] = NDPI_PROTOCOL_UNKNOWN; + } + } else { + if((!ndpi_struct->http_dont_dissect_response) && (flow->http.response_status_code[0] == '\0')) { + flow->http_upper_protocol = flow->detected_protocol_stack[0], flow->http_lower_protocol = flow->detected_protocol_stack[1]; + flow->detected_protocol_stack[0] = NDPI_PROTOCOL_UNKNOWN, flow->detected_protocol_stack[1] = NDPI_PROTOCOL_UNKNOWN; + flow->packet.detected_protocol_stack[0] = NDPI_PROTOCOL_UNKNOWN, flow->packet.detected_protocol_stack[1] = NDPI_PROTOCOL_UNKNOWN; + } } + + flow->http_detected = 1, flow->guessed_category = category; } #ifdef NDPI_CONTENT_CATEGORY_FLASH @@ -79,7 +93,7 @@ static void avi_check_http_payload(struct ndpi_detection_module_struct *ndpi_str NDPI_LOG_DBG2(ndpi_struct, "called avi_check_http_payload: %u %u %u\n", - packet->empty_line_position_set, flow->l4.tcp.http_empty_line_seen, packet->empty_line_position); + packet->empty_line_position_set, flow->l4.tcp.http_empty_line_seen, packet->empty_line_position); if(packet->empty_line_position_set == 0 && flow->l4.tcp.http_empty_line_seen == 0) return; @@ -125,7 +139,7 @@ static void teamviewer_check_http_payload(struct ndpi_detection_module_struct *n const u_int8_t *pos; NDPI_LOG_DBG2(ndpi_struct, "called teamviewer_check_http_payload: %u %u %u\n", - packet->empty_line_position_set, flow->l4.tcp.http_empty_line_seen, packet->empty_line_position); + packet->empty_line_position_set, flow->l4.tcp.http_empty_line_seen, packet->empty_line_position); if(packet->empty_line_position_set == 0 || (packet->empty_line_position + 5) > (packet->payload_packet_len)) return; @@ -171,10 +185,10 @@ static void setHttpUserAgent(struct ndpi_detection_module_struct *ndpi_struct, static void parseHttpSubprotocol(struct ndpi_detection_module_struct *ndpi_struct, struct ndpi_flow_struct *flow) { if((flow->l4.tcp.http_stage == 0) || (flow->http.url && flow->http_detected)) { - char *double_col = strchr((char*)flow->host_server_name, ':'); - ndpi_protocol_match_result ret_match; - - if(double_col) double_col[0] = '\0'; + char *double_col = strchr((char*)flow->host_server_name, ':'); + ndpi_protocol_match_result ret_match; + + if(double_col) double_col[0] = '\0'; /** NOTE @@ -198,6 +212,13 @@ static void check_content_type_and_change_protocol(struct ndpi_detection_module_ struct ndpi_packet_struct *packet = &flow->packet; u_int8_t a; + if((!ndpi_struct->http_dont_dissect_response) && flow->http_detected && (flow->http.response_status_code[0] != 0)) { + ndpi_set_detected_protocol(ndpi_struct, flow, flow->http_upper_protocol, flow->http_lower_protocol); +#ifdef DEBUG + printf("[%s] [http_dont_dissect_response: %u]->> %s\n", __FUNCTION__, ndpi_struct->http_dont_dissect_response, flow->http.response_status_code); +#endif + return; + } #if defined(NDPI_PROTOCOL_1KXUN) || defined(NDPI_PROTOCOL_IQIYI) /* PPStream */ @@ -277,8 +298,8 @@ static void check_content_type_and_change_protocol(struct ndpi_detection_module_ if(packet->user_agent_line.ptr != NULL && packet->user_agent_line.len != 0) { /** Format examples: - Mozilla/5.0 (iPad; U; CPU OS 3_2 like Mac OS X; en-us) AppleWebKit/531.21.10 (KHTML, like Gecko) .... - Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:54.0) Gecko/20100101 Firefox/54.0 + Mozilla/5.0 (iPad; U; CPU OS 3_2 like Mac OS X; en-us) AppleWebKit/531.21.10 (KHTML, like Gecko) .... + Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:54.0) Gecko/20100101 Firefox/54.0 */ if(packet->user_agent_line.len > 7) { char ua[256]; @@ -308,22 +329,22 @@ static void check_content_type_and_change_protocol(struct ndpi_detection_module_ if(token && (token[0] == ' ')) token++; /* Skip space */ if(token - && ((strcmp(token, "U") == 0) - || (strncmp(token, "MSIE", 4) == 0))) { - token = strsep(&parent, ";"); - if(token && (token[0] == ' ')) token++; /* Skip space */ + && ((strcmp(token, "U") == 0) + || (strncmp(token, "MSIE", 4) == 0))) { + token = strsep(&parent, ";"); + if(token && (token[0] == ' ')) token++; /* Skip space */ - if(token && (strncmp(token, "Update", 6) == 0)) { - token = strsep(&parent, ";"); + if(token && (strncmp(token, "Update", 6) == 0)) { + token = strsep(&parent, ";"); - if(token && (token[0] == ' ')) token++; /* Skip space */ + if(token && (token[0] == ' ')) token++; /* Skip space */ - if(token && (strncmp(token, "AOL", 3) == 0)) { + if(token && (strncmp(token, "AOL", 3) == 0)) { - token = strsep(&parent, ";"); - if(token && (token[0] == ' ')) token++; /* Skip space */ - } - } + token = strsep(&parent, ";"); + if(token && (token[0] == ' ')) token++; /* Skip space */ + } + } } } @@ -340,7 +361,7 @@ static void check_content_type_and_change_protocol(struct ndpi_detection_module_ } NDPI_LOG_DBG2(ndpi_struct, "User Agent Type line found %.*s\n", - packet->user_agent_line.len, packet->user_agent_line.ptr); + packet->user_agent_line.len, packet->user_agent_line.ptr); } /* check for host line */ @@ -348,37 +369,37 @@ static void check_content_type_and_change_protocol(struct ndpi_detection_module_ u_int len; NDPI_LOG_DBG2(ndpi_struct, "HOST line found %.*s\n", - packet->host_line.len, packet->host_line.ptr); + packet->host_line.len, packet->host_line.ptr); /* call ndpi_match_host_subprotocol to see if there is a match with known-host HTTP subprotocol */ if((ndpi_struct->http_dont_dissect_response) || flow->http_detected) { ndpi_protocol_match_result ret_match; - + ndpi_match_host_subprotocol(ndpi_struct, flow, (char*)packet->host_line.ptr, packet->host_line.len, &ret_match, NDPI_PROTOCOL_HTTP); } - + /* Copy result for nDPI apps */ if(!ndpi_struct->disable_metadata_export) { len = ndpi_min(packet->host_line.len, sizeof(flow->host_server_name)-1); strncpy((char*)flow->host_server_name, (char*)packet->host_line.ptr, len); flow->host_server_name[len] = '\0'; } - + flow->server_id = flow->dst; if(packet->forwarded_line.ptr) { - len = ndpi_min(packet->forwarded_line.len, sizeof(flow->protos.http.nat_ip)-1); - if(!ndpi_struct->disable_metadata_export) { - strncpy((char*)flow->protos.http.nat_ip, (char*)packet->forwarded_line.ptr, len); - flow->protos.http.nat_ip[len] = '\0'; - } + len = ndpi_min(packet->forwarded_line.len, sizeof(flow->protos.http.nat_ip)-1); + if(!ndpi_struct->disable_metadata_export) { + strncpy((char*)flow->protos.http.nat_ip, (char*)packet->forwarded_line.ptr, len); + flow->protos.http.nat_ip[len] = '\0'; + } } - if(ndpi_struct->http_dont_dissect_response) + if(!ndpi_struct->http_dont_dissect_response) parseHttpSubprotocol(ndpi_struct, flow); /** @@ -403,26 +424,28 @@ static void check_content_type_and_change_protocol(struct ndpi_detection_module_ && ((ndpi_struct->http_dont_dissect_response) || flow->http_detected) && (packet->http_origin.len > 0)) { ndpi_protocol_match_result ret_match; - + ndpi_match_host_subprotocol(ndpi_struct, flow, (char *)packet->http_origin.ptr, packet->http_origin.len, &ret_match, NDPI_PROTOCOL_HTTP); } - + if(flow->detected_protocol_stack[0] != NDPI_PROTOCOL_UNKNOWN) { if(packet->detected_protocol_stack[0] != NDPI_PROTOCOL_HTTP) { - NDPI_LOG_INFO(ndpi_struct, "found HTTP/%s\n", - ndpi_get_proto_name(ndpi_struct, packet->detected_protocol_stack[0])); + NDPI_LOG_INFO(ndpi_struct, "found HTTP/%s\n", + ndpi_get_proto_name(ndpi_struct, packet->detected_protocol_stack[0])); ndpi_int_http_add_connection(ndpi_struct, flow, packet->detected_protocol_stack[0]); return; /* We have identified a sub-protocol so we're done */ } } } +#if 0 if(!ndpi_struct->http_dont_dissect_response && flow->http_detected) parseHttpSubprotocol(ndpi_struct, flow); +#endif if(flow->guessed_protocol_id == NDPI_PROTOCOL_UNKNOWN) flow->guessed_protocol_id = NDPI_PROTOCOL_HTTP; @@ -430,7 +453,7 @@ static void check_content_type_and_change_protocol(struct ndpi_detection_module_ /* check for accept line */ if(packet->accept_line.ptr != NULL) { NDPI_LOG_DBG2(ndpi_struct, "Accept line found %.*s\n", - packet->accept_line.len, packet->accept_line.ptr); + packet->accept_line.len, packet->accept_line.ptr); if(NDPI_COMPARE_PROTOCOL_TO_BITMASK(ndpi_struct->detection_bitmask, NDPI_PROTOCOL_RTSP) != 0) { rtsp_parse_packet_acceptline(ndpi_struct, flow); @@ -438,26 +461,28 @@ static void check_content_type_and_change_protocol(struct ndpi_detection_module_ } /* search for line startin with "Icy-MetaData" */ - for (a = 0; a < packet->parsed_lines; a++) { + for (a = 0; a < packet->parsed_lines; a++) { if(packet->line[a].len > 11 && memcmp(packet->line[a].ptr, "Icy-MetaData", 12) == 0) { - NDPI_LOG_INFO(ndpi_struct, "found MPEG: Icy-MetaData\n"); - ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_CONTENT_CATEGORY_MPEG); - return; - } + NDPI_LOG_INFO(ndpi_struct, "found MPEG: Icy-MetaData\n"); + ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_CONTENT_CATEGORY_MPEG); + return; + } } if(packet->content_line.ptr != NULL && packet->content_line.len != 0) { NDPI_LOG_DBG2(ndpi_struct, "Content Type line found %.*s\n", - packet->content_line.len, packet->content_line.ptr); + packet->content_line.len, packet->content_line.ptr); if((ndpi_struct->http_dont_dissect_response) || flow->http_detected) { ndpi_protocol_match_result ret_match; - + ndpi_match_content_subprotocol(ndpi_struct, flow, (char*)packet->content_line.ptr, packet->content_line.len, &ret_match, NDPI_PROTOCOL_HTTP); } } + + ndpi_int_http_add_connection(ndpi_struct, flow, packet->detected_protocol_stack[0]); } static void check_http_payload(struct ndpi_detection_module_struct *ndpi_struct, struct ndpi_flow_struct *flow) @@ -486,22 +511,22 @@ static void check_http_payload(struct ndpi_detection_module_struct *ndpi_struct, #define STATIC_STRING_L(a) {.str=a, .len=sizeof(a)-1 } static struct l_string { - const char *str; - size_t len; + const char *str; + size_t len; } http_methods[] = { - STATIC_STRING_L("GET "), - STATIC_STRING_L("POST "), - STATIC_STRING_L("OPTIONS "), - STATIC_STRING_L("HEAD "), - STATIC_STRING_L("PUT "), - STATIC_STRING_L("DELETE "), - STATIC_STRING_L("CONNECT "), - STATIC_STRING_L("PROPFIND "), - STATIC_STRING_L("REPORT ") }; + STATIC_STRING_L("GET "), + STATIC_STRING_L("POST "), + STATIC_STRING_L("OPTIONS "), + STATIC_STRING_L("HEAD "), + STATIC_STRING_L("PUT "), + STATIC_STRING_L("DELETE "), + STATIC_STRING_L("CONNECT "), + STATIC_STRING_L("PROPFIND "), + STATIC_STRING_L("REPORT ") }; static const char *http_fs = "CDGHOPR"; static uint8_t non_ctrl(uint8_t c) { - return c < 32 ? '.':c; + return c < 32 ? '.':c; } static u_int16_t http_request_url_offset(struct ndpi_detection_module_struct *ndpi_struct, struct ndpi_flow_struct *flow) @@ -510,9 +535,9 @@ static u_int16_t http_request_url_offset(struct ndpi_detection_module_struct *nd int i; NDPI_LOG_DBG2(ndpi_struct, "====>>>> HTTP: %c%c%c%c [len: %u]\n", - non_ctrl(packet->payload[0]), non_ctrl(packet->payload[1]), - non_ctrl(packet->payload[2]), non_ctrl(packet->payload[3]), - packet->payload_packet_len); + non_ctrl(packet->payload[0]), non_ctrl(packet->payload[1]), + non_ctrl(packet->payload[2]), non_ctrl(packet->payload[3]), + packet->payload_packet_len); /* Check first char */ if(!strchr(http_fs,packet->payload[0])) return 0; @@ -520,11 +545,11 @@ static u_int16_t http_request_url_offset(struct ndpi_detection_module_struct *nd FIRST PAYLOAD PACKET FROM CLIENT **/ for(i=0; i < sizeof(http_methods)/sizeof(http_methods[0]); i++) { - if(packet->payload_packet_len >= http_methods[i].len && - memcmp(packet->payload,http_methods[i].str,http_methods[i].len) == 0) { - NDPI_LOG_DBG2(ndpi_struct, "HTTP: %sFOUND\n",http_methods[i].str); - return http_methods[i].len; - } + if(packet->payload_packet_len >= http_methods[i].len && + memcmp(packet->payload,http_methods[i].str,http_methods[i].len) == 0) { + NDPI_LOG_DBG2(ndpi_struct, "HTTP: %sFOUND\n",http_methods[i].str); + return http_methods[i].len; + } } return 0; } @@ -556,7 +581,7 @@ static void http_bitmask_exclude_other(struct ndpi_flow_struct *flow) /*************************************************************************************************/ static void ndpi_check_http_tcp(struct ndpi_detection_module_struct *ndpi_struct, - struct ndpi_flow_struct *flow) { + struct ndpi_flow_struct *flow) { struct ndpi_packet_struct *packet = &flow->packet; u_int16_t filename_start; /* the filename in the request method line, e.g., "GET filename_start..."*/ @@ -586,7 +611,7 @@ static void ndpi_check_http_tcp(struct ndpi_detection_module_struct *ndpi_struct flow->l4.tcp.http_stage = 1; return; } - + if((packet->payload_packet_len == 40) && (flow->l4.tcp.http_stage == 0)) { /* -> QR O06L0072-6L91-4O43-857J-K8OO172L6L51 @@ -600,11 +625,11 @@ static void ndpi_check_http_tcp(struct ndpi_detection_module_struct *ndpi_struct && (packet->payload[21] == '-') && (packet->payload[26] == '-') && (packet->payload[39] == 0x0A) - ) - flow->l4.tcp.http_stage = 1; - return; + ) + flow->l4.tcp.http_stage = 1; + return; } - + if((packet->payload_packet_len == 23) && (memcmp(packet->payload, "", 23) == 0)) { /* @@ -617,8 +642,8 @@ static void ndpi_check_http_tcp(struct ndpi_detection_module_struct *ndpi_struct ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_OOKLA, NDPI_PROTOCOL_UNKNOWN); if(ndpi_struct->ookla_cache == NULL) - ndpi_struct->ookla_cache = lruc_new(4*1024, 1024); - + ndpi_struct->ookla_cache = lruc_new(4*1024, 1024); + if(ndpi_struct->ookla_cache != NULL) { u_int8_t *dummy = (u_int8_t*)ndpi_malloc(sizeof(u_int8_t)); @@ -629,23 +654,23 @@ static void ndpi_check_http_tcp(struct ndpi_detection_module_struct *ndpi_struct lruc_set((lruc*)ndpi_struct->ookla_cache, (void*)&packet->iph->daddr, 4, dummy, 1); } } - + return; } - + NDPI_EXCLUDE_PROTO(ndpi_struct, flow); http_bitmask_exclude_other(flow); return; } NDPI_LOG_DBG2(ndpi_struct, - "Filename HTTP found: %d, we look for line info..\n", filename_start); + "Filename HTTP found: %d, we look for line info..\n", filename_start); ndpi_parse_packet_line_info(ndpi_struct, flow); if(packet->parsed_lines <= 1) { NDPI_LOG_DBG2(ndpi_struct, - "Found just one line, we will look further for the next packet...\n"); + "Found just one line, we will look further for the next packet...\n"); packet->http_method.ptr = packet->line[0].ptr; packet->http_method.len = filename_start - 1; @@ -656,10 +681,10 @@ static void ndpi_check_http_tcp(struct ndpi_detection_module_struct *ndpi_struct } NDPI_LOG_DBG2(ndpi_struct, - "Found more than one line, we look further for the next packet...\n"); + "Found more than one line, we look further for the next packet...\n"); if(packet->line[0].len >= (9 + filename_start) - && memcmp(&packet->line[0].ptr[packet->line[0].len - 9], " HTTP/1.", 8) == 0) { /* Request line complete. Ex. "GET / HTTP/1.1" */ + && memcmp(&packet->line[0].ptr[packet->line[0].len - 9], " HTTP/1.", 8) == 0) { /* Request line complete. Ex. "GET / HTTP/1.1" */ packet->http_url_name.ptr = &packet->payload[filename_start]; packet->http_url_name.len = packet->line[0].len - (filename_start + 9); @@ -669,14 +694,13 @@ static void ndpi_check_http_tcp(struct ndpi_detection_module_struct *ndpi_struct // Set the HTTP requested version: 0=HTTP/1.0 and 1=HTTP/1.1 if(memcmp(&packet->line[0].ptr[packet->line[0].len - 1], "1", 1) == 0) - flow->http.request_version = 1; + flow->http.request_version = 1; else - flow->http.request_version = 0; + flow->http.request_version = 0; /* Set the first found headers in request */ flow->http.num_request_headers = packet->http_num_headers; - /* Check for Ookla */ if((packet->referer_line.len > 0) && ndpi_strnstr((const char *)packet->referer_line.ptr, "www.speedtest.net", packet->referer_line.len)) { @@ -686,22 +710,22 @@ static void ndpi_check_http_tcp(struct ndpi_detection_module_struct *ndpi_struct /* Check for additional field introduced by Steam */ int x = 1; if(packet->line[x].len >= 11 && (memcmp(packet->line[x].ptr, "x-steam-sid", 11)) == 0) { - NDPI_LOG_INFO(ndpi_struct, "found STEAM\n"); - ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_STEAM); - check_content_type_and_change_protocol(ndpi_struct, flow); - return; + NDPI_LOG_INFO(ndpi_struct, "found STEAM\n"); + ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_STEAM); + check_content_type_and_change_protocol(ndpi_struct, flow); + return; } /* Check for additional field introduced by Facebook */ x = 1; while(packet->line[x].len != 0) { - if(packet->line[x].len >= 12 && (memcmp(packet->line[x].ptr, "X-FB-SIM-HNI", 12)) == 0) { - NDPI_LOG_INFO(ndpi_struct, "found FACEBOOK\n"); - ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_FACEBOOK); - check_content_type_and_change_protocol(ndpi_struct, flow); - return; - } - x++; + if(packet->line[x].len >= 12 && (memcmp(packet->line[x].ptr, "X-FB-SIM-HNI", 12)) == 0) { + NDPI_LOG_INFO(ndpi_struct, "found FACEBOOK\n"); + ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_FACEBOOK); + check_content_type_and_change_protocol(ndpi_struct, flow); + return; + } + x++; } #if defined(NDPI_PROTOCOL_1KXUN) || defined(NDPI_PROTOCOL_IQIYI) @@ -746,23 +770,23 @@ static void ndpi_check_http_tcp(struct ndpi_detection_module_struct *ndpi_struct } } #endif - + if((packet->http_url_name.len > 7) - && (!strncmp((const char*) packet->http_url_name.ptr, "http://", 7))) { + && (!strncmp((const char*) packet->http_url_name.ptr, "http://", 7))) { NDPI_LOG_INFO(ndpi_struct, "found HTTP_PROXY\n"); ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_HTTP_PROXY); check_content_type_and_change_protocol(ndpi_struct, flow); } if(filename_start == 8 && (memcmp(packet->payload, "CONNECT ", 8) == 0)) { - /* nathan@getoffmalawn.com */ + /* nathan@getoffmalawn.com */ NDPI_LOG_INFO(ndpi_struct, "found HTTP_CONNECT\n"); ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_HTTP_CONNECT); check_content_type_and_change_protocol(ndpi_struct, flow); } NDPI_LOG_DBG2(ndpi_struct, - "HTTP START Found, we will look for sub-protocols (content and host)...\n"); + "HTTP START Found, we will look for sub-protocols (content and host)...\n"); if(packet->host_line.ptr != NULL) { /** @@ -775,11 +799,11 @@ static void ndpi_check_http_tcp(struct ndpi_detection_module_struct *ndpi_struct if(ndpi_struct->http_dont_dissect_response) { if(flow->detected_protocol_stack[0] == NDPI_PROTOCOL_UNKNOWN) /* No subprotocol found */ NDPI_LOG_INFO(ndpi_struct, "found HTTP\n"); - ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_HTTP); + ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_HTTP); } else { flow->http_detected = 1; NDPI_LOG_DBG2(ndpi_struct, - "HTTP START Found, we will look further for the response...\n"); + "HTTP START Found, we will look further for the response...\n"); flow->l4.tcp.http_stage = packet->packet_direction + 1; // packet_direction 0: stage 1, packet_direction 1: stage 2 } @@ -790,23 +814,21 @@ static void ndpi_check_http_tcp(struct ndpi_detection_module_struct *ndpi_struct NDPI_EXCLUDE_PROTO(ndpi_struct, flow); http_bitmask_exclude_other(flow); - } else if((flow->l4.tcp.http_stage == 1) || (flow->l4.tcp.http_stage == 2)) { - NDPI_LOG_DBG2(ndpi_struct, "HTTP stage %u: \n", flow->l4.tcp.http_stage); - + if((packet->payload_packet_len == 34) && (flow->l4.tcp.http_stage == 1)) { if((packet->payload[5] == ' ') && (packet->payload[9] == ' ')) { goto ookla_found; } } - + if((packet->payload_packet_len > 6) && memcmp(packet->payload, "HELLO ", 6) == 0) { - /* This looks like Ookla */ + /* This looks like Ookla */ goto ookla_found; } else - NDPI_ADD_PROTOCOL_TO_BITMASK(flow->excluded_protocol_bitmask, NDPI_PROTOCOL_OOKLA); - + NDPI_ADD_PROTOCOL_TO_BITMASK(flow->excluded_protocol_bitmask, NDPI_PROTOCOL_OOKLA); + /** At first check, if this is for sure a response packet (in another direction. If not, if HTTP is detected do nothing now and return, otherwise check the second packet for the HTTP request @@ -817,7 +839,7 @@ static void ndpi_check_http_tcp(struct ndpi_detection_module_struct *ndpi_struct return; NDPI_LOG_DBG2(ndpi_struct, - " SECOND PAYLOAD TRAFFIC FROM CLIENT, FIRST PACKET MIGHT HAVE BEEN HTTP...UNKNOWN TRAFFIC, HERE FOR HTTP again.. \n"); + " SECOND PAYLOAD TRAFFIC FROM CLIENT, FIRST PACKET MIGHT HAVE BEEN HTTP...UNKNOWN TRAFFIC, HERE FOR HTTP again.. \n"); ndpi_parse_packet_line_info(ndpi_struct, flow); @@ -839,14 +861,14 @@ static void ndpi_check_http_tcp(struct ndpi_detection_module_struct *ndpi_struct } // http://www.slideshare.net/DSPIP/rtsp-analysis-wireshark if(packet->line[0].len >= 9 - && memcmp(&packet->line[0].ptr[packet->line[0].len - 9], " HTTP/1.", 8) == 0) { + && memcmp(&packet->line[0].ptr[packet->line[0].len - 9], " HTTP/1.", 8) == 0) { NDPI_LOG_INFO(ndpi_struct, "found HTTP\n"); ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_HTTP); check_content_type_and_change_protocol(ndpi_struct, flow); NDPI_LOG_DBG2(ndpi_struct, - "HTTP START Found in 2. packet, we will look further for the response....\n"); + "HTTP START Found in 2. packet, we will look further for the response....\n"); flow->http_detected = 1; } @@ -879,9 +901,8 @@ static void ndpi_check_http_tcp(struct ndpi_detection_module_struct *ndpi_struct ndpi_parse_packet_line_info(ndpi_struct, flow); check_content_type_and_change_protocol(ndpi_struct, flow); - if(packet->packet_direction == 1 /* server -> client */){ - flow->http.num_response_headers += packet->http_num_headers; /* flow structs are initialized with zeros */ - } + if(packet->packet_direction == 1 /* server -> client */) + flow->http.num_response_headers += packet->http_num_headers; /* flow structs are initialized with zeros */ if(packet->empty_line_position_set != 0 || flow->l4.tcp.http_empty_line_seen == 1) { NDPI_LOG_DBG2(ndpi_struct, "empty line. check_http_payload\n"); @@ -905,8 +926,8 @@ void ndpi_search_http_tcp(struct ndpi_detection_module_struct *ndpi_struct, } if(packet->detected_protocol_stack[0] != NDPI_PROTOCOL_UNKNOWN) { - return; - } + return; + } NDPI_LOG_DBG(ndpi_struct, "search HTTP\n"); ndpi_check_http_tcp(ndpi_struct, flow); @@ -935,7 +956,7 @@ char* ndpi_get_http_url(struct ndpi_detection_module_struct *ndpi_mod, /* ********************************* */ char* ndpi_get_http_content_type(struct ndpi_detection_module_struct *ndpi_mod, - struct ndpi_flow_struct *flow) { + struct ndpi_flow_struct *flow) { if((!flow) || (!flow->http.content_type)) return(""); else -- cgit v1.2.3 From f2f2dbe381c0f92f0cc859835238723fea77c767 Mon Sep 17 00:00:00 2001 From: Luca Deri Date: Mon, 14 Jan 2019 20:03:06 +0100 Subject: Fix for IPv6 HTTP traffic --- src/lib/protocols/http.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/lib/protocols/http.c') diff --git a/src/lib/protocols/http.c b/src/lib/protocols/http.c index e92701072..b1917281c 100644 --- a/src/lib/protocols/http.c +++ b/src/lib/protocols/http.c @@ -630,7 +630,7 @@ static void ndpi_check_http_tcp(struct ndpi_detection_module_struct *ndpi_struct return; } - if((packet->payload_packet_len == 23) && (memcmp(packet->payload, "", 23) == 0)) { + if((packet->iph != NULL) && (packet->payload_packet_len == 23) && (memcmp(packet->payload, "", 23) == 0)) { /* -- cgit v1.2.3 From 2ade0be365429a404d63f8ee20f1b59650cde710 Mon Sep 17 00:00:00 2001 From: Simone Mainardi Date: Tue, 15 Jan 2019 09:56:33 +0100 Subject: Fixes ookla lru cache use with IPv6 --- src/lib/protocols/http.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src/lib/protocols/http.c') diff --git a/src/lib/protocols/http.c b/src/lib/protocols/http.c index b1917281c..661e55732 100644 --- a/src/lib/protocols/http.c +++ b/src/lib/protocols/http.c @@ -630,7 +630,7 @@ static void ndpi_check_http_tcp(struct ndpi_detection_module_struct *ndpi_struct return; } - if((packet->iph != NULL) && (packet->payload_packet_len == 23) && (memcmp(packet->payload, "", 23) == 0)) { + if((packet->payload_packet_len == 23) && (memcmp(packet->payload, "", 23) == 0)) { /* @@ -644,7 +644,7 @@ static void ndpi_check_http_tcp(struct ndpi_detection_module_struct *ndpi_struct if(ndpi_struct->ookla_cache == NULL) ndpi_struct->ookla_cache = lruc_new(4*1024, 1024); - if(ndpi_struct->ookla_cache != NULL) { + if(packet->iph != NULL && ndpi_struct->ookla_cache != NULL) { u_int8_t *dummy = (u_int8_t*)ndpi_malloc(sizeof(u_int8_t)); if(dummy) { -- cgit v1.2.3 From efef99cbadc8ddd6f6743e04d184fe240d6eb334 Mon Sep 17 00:00:00 2001 From: Luca Deri Date: Thu, 17 Jan 2019 00:40:28 +0100 Subject: Removed this party LRU and replaced with home grown --- example/ndpiReader.c | 5 +- src/include/Makefile.am | 2 +- src/include/ndpi_api.h | 6 + src/include/ndpi_typedefs.h | 6 +- src/lib/ndpi_main.c | 48 +++++- src/lib/protocols/http.c | 19 +-- src/lib/protocols/ookla.c | 17 +-- src/lib/protocols/ssl.c | 4 + src/lib/third_party/include/lruc.h | 55 ------- src/lib/third_party/src/lruc.c | 294 ------------------------------------- 10 files changed, 79 insertions(+), 377 deletions(-) delete mode 100644 src/lib/third_party/include/lruc.h delete mode 100644 src/lib/third_party/src/lruc.c (limited to 'src/lib/protocols/http.c') diff --git a/example/ndpiReader.c b/example/ndpiReader.c index 6c3dfeee8..8996ea1a5 100644 --- a/example/ndpiReader.c +++ b/example/ndpiReader.c @@ -761,6 +761,9 @@ static void printFlow(u_int16_t id, struct ndpi_flow_info *flow, u_int16_t threa if((verbose != 1) && (verbose != 2)) return; + if(5222 == ntohs(flow->dst_port)) + printf("************\n"); + if(!json_flag) { fprintf(out, "\t%u", id); @@ -913,7 +916,7 @@ static void node_proto_guess_walker(const void *node, ndpi_VISIT which, int dept struct ndpi_flow_info *flow = *(struct ndpi_flow_info **) node; u_int16_t thread_id = *((u_int16_t *) user_data); - if((which == ndpi_preorder) || (which == ndpi_leaf)) { /* Avoid walking the same node multiple times */ + if((which == ndpi_preorder) || (which == ndpi_leaf)) { /* Avoid walking the same node multiple times */ if((!flow->detection_completed) && flow->ndpi_flow) flow->detected_protocol = ndpi_detection_giveup(ndpi_thread_info[0].workflow->ndpi_struct, flow->ndpi_flow, enable_protocol_guess); diff --git a/src/include/Makefile.am b/src/include/Makefile.am index 6eeca93b4..47fcbd224 100644 --- a/src/include/Makefile.am +++ b/src/include/Makefile.am @@ -1,4 +1,4 @@ -library_includedir=$(includedir)/libndpi-@VERSION@/libndpi +plibrary_includedir=$(includedir)/libndpi-@VERSION@/libndpi library_include_HEADERS = ndpi_api.h \ ndpi_define.h \ diff --git a/src/include/ndpi_api.h b/src/include/ndpi_api.h index e57f3a568..e09c91c8e 100644 --- a/src/include/ndpi_api.h +++ b/src/include/ndpi_api.h @@ -756,6 +756,12 @@ extern "C" { u_int ndpi_get_ndpi_num_custom_protocols(struct ndpi_detection_module_struct *ndpi_mod); u_int ndpi_get_ndpi_detection_module_size(); void ndpi_set_log_level(struct ndpi_detection_module_struct *ndpi_mod, u_int l); + + /* LRU cache */ + struct ndpi_lru_cache* ndpi_lru_cache_init(u_int32_t num_entries); + void ndpi_lru_free_cache(struct ndpi_lru_cache *c); + u_int8_t ndpi_lru_find_cache(struct ndpi_lru_cache *c, u_int32_t key, u_int8_t clean_key_when_found); + void ndpi_lru_add_to_cache(struct ndpi_lru_cache *c, u_int32_t key); /** * Add a string to match to an automata diff --git a/src/include/ndpi_typedefs.h b/src/include/ndpi_typedefs.h index 34d308e2e..cf1af8bc3 100644 --- a/src/include/ndpi_typedefs.h +++ b/src/include/ndpi_typedefs.h @@ -367,6 +367,10 @@ typedef enum { HTTP_METHOD_CONNECT } ndpi_http_method; +struct ndpi_lru_cache { + u_int32_t num_entries, *entries; +}; + struct ndpi_id_struct { /** detected_protocol_bitmask: @@ -997,7 +1001,7 @@ struct ndpi_detection_module_struct { int bt_ann_len; /* NDPI_PROTOCOL_OOKLA */ - void *ookla_cache; + struct ndpi_lru_cache *ookla_cache; /* NDPI_PROTOCOL_TINC */ struct cache *tinc_cache; diff --git a/src/lib/ndpi_main.c b/src/lib/ndpi_main.c index 863943c4a..594d521d3 100644 --- a/src/lib/ndpi_main.c +++ b/src/lib/ndpi_main.c @@ -30,7 +30,6 @@ #include #include "ahocorasick.h" #include "libcache.h" -#include "lruc.h" #define NDPI_CURRENT_PROTO NDPI_PROTOCOL_UNKNOWN @@ -2433,7 +2432,7 @@ void ndpi_exit_detection_module(struct ndpi_detection_module_struct *ndpi_struct cache_free((cache_t)(ndpi_struct->tinc_cache)); if(ndpi_struct->ookla_cache) - lruc_free((lruc*)ndpi_struct->ookla_cache); + ndpi_lru_free_cache(ndpi_struct->ookla_cache); if(ndpi_struct->protocols_ptree) ndpi_Destroy_Patricia((patricia_tree_t*)ndpi_struct->protocols_ptree, free_ptree_data); @@ -3995,6 +3994,9 @@ ndpi_protocol ndpi_detection_giveup(struct ndpi_detection_module_struct *ndpi_st if(flow == NULL) return(ret); + if(flow->packet.tcp && (5222 == ntohs(flow->packet.tcp->dest))) + printf("%u - %u\n", ntohs(flow->packet.tcp->source), ntohs(flow->packet.tcp->dest)); + /* TODO: add the remaining stage_XXXX protocols */ if(flow->detected_protocol_stack[0] == NDPI_PROTOCOL_UNKNOWN) { u_int16_t guessed_protocol_id, guessed_host_protocol_id; @@ -6171,6 +6173,48 @@ void ndpi_set_log_level(struct ndpi_detection_module_struct *ndpi_mod, u_int l) /* ******************************************************************** */ +/* LRU cache */ + +struct ndpi_lru_cache* ndpi_lru_cache_init(u_int32_t num_entries) { + struct ndpi_lru_cache *c = (struct ndpi_lru_cache*)malloc(sizeof(struct ndpi_lru_cache)); + + if(!c) return(NULL); + + c->entries = (u_int32_t*)calloc(num_entries, sizeof(u_int32_t)); + + if(!c->entries) { + free(c); + return(NULL); + } else + c->num_entries = num_entries; + + return(c); +} + +void ndpi_lru_free_cache(struct ndpi_lru_cache *c) { + free(c->entries); + free(c); +} + + +u_int8_t ndpi_lru_find_cache(struct ndpi_lru_cache *c, u_int32_t key, u_int8_t clean_key_when_found) { + u_int32_t slot = key % c->num_entries; + + if(c->entries[slot] == key) { + if(clean_key_when_found) c->entries[slot] = 0; + return(1); + } else + return(0); +} + +void ndpi_lru_add_to_cache(struct ndpi_lru_cache *c, u_int32_t key) { + u_int32_t slot = key % c->num_entries; + + c->entries[slot] = key; +} + +/* ******************************************************************** */ + /* NOTE: - Leave fields empty/zero when information is missing (e.g. with ICMP ports are zero) diff --git a/src/lib/protocols/http.c b/src/lib/protocols/http.c index 661e55732..16b122d06 100644 --- a/src/lib/protocols/http.c +++ b/src/lib/protocols/http.c @@ -26,7 +26,6 @@ #define NDPI_CURRENT_PROTO NDPI_PROTOCOL_HTTP #include "ndpi_api.h" -#include "lruc.h" static void ndpi_int_http_add_connection(struct ndpi_detection_module_struct *ndpi_struct, struct ndpi_flow_struct *flow, @@ -642,19 +641,15 @@ static void ndpi_check_http_tcp(struct ndpi_detection_module_struct *ndpi_struct ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_OOKLA, NDPI_PROTOCOL_UNKNOWN); if(ndpi_struct->ookla_cache == NULL) - ndpi_struct->ookla_cache = lruc_new(4*1024, 1024); - + ndpi_struct->ookla_cache = ndpi_lru_cache_init(1024); + if(packet->iph != NULL && ndpi_struct->ookla_cache != NULL) { - u_int8_t *dummy = (u_int8_t*)ndpi_malloc(sizeof(u_int8_t)); - - if(dummy) { - if(packet->tcp->source == htons(8080)) - lruc_set((lruc*)ndpi_struct->ookla_cache, (void*)&packet->iph->saddr, 4, dummy, 1); - else - lruc_set((lruc*)ndpi_struct->ookla_cache, (void*)&packet->iph->daddr, 4, dummy, 1); - } + if(packet->tcp->source == htons(8080)) + ndpi_lru_add_to_cache(ndpi_struct->ookla_cache, packet->iph->saddr); + else + ndpi_lru_add_to_cache(ndpi_struct->ookla_cache, packet->iph->daddr); } - + return; } diff --git a/src/lib/protocols/ookla.c b/src/lib/protocols/ookla.c index 4d46dbf58..06d97e216 100644 --- a/src/lib/protocols/ookla.c +++ b/src/lib/protocols/ookla.c @@ -22,36 +22,32 @@ #define NDPI_CURRENT_PROTO NDPI_PROTOCOL_OOKLA #include "ndpi_api.h" -#include "lruc.h" void ndpi_search_ookla(struct ndpi_detection_module_struct* ndpi_struct, struct ndpi_flow_struct* flow) { -#if 0 struct ndpi_packet_struct* packet = &flow->packet; u_int32_t addr = 0; void *value; - + NDPI_LOG_DBG(ndpi_struct, "Ookla detection\n"); - + if(packet->tcp->source == htons(8080)) addr = packet->iph->saddr; else if(packet->tcp->dest == htons(8080)) addr = packet->iph->daddr; else goto ookla_exclude; - + if(ndpi_struct->ookla_cache != NULL) { - if(lruc_get(ndpi_struct->ookla_cache, &addr, sizeof(addr), &value) == LRUC_NO_ERROR) { - /* Don't remove it as it can be used for other connections */ + if(ndpi_lru_find_cache(ndpi_struct->ookla_cache, addr, 0 /* Don't remove it as it can be used for other connections */)) { NDPI_LOG_INFO(ndpi_struct, "found ookla tcp connection\n"); ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_OOKLA, NDPI_PROTOCOL_UNKNOWN); return; - } + } } ookla_exclude: -#endif - NDPI_EXCLUDE_PROTO(ndpi_struct, flow); + NDPI_EXCLUDE_PROTO(ndpi_struct, flow); } void init_ookla_dissector(struct ndpi_detection_module_struct *ndpi_struct, @@ -65,4 +61,3 @@ void init_ookla_dissector(struct ndpi_detection_module_struct *ndpi_struct, *id += 1; } - diff --git a/src/lib/protocols/ssl.c b/src/lib/protocols/ssl.c index 25d535a57..785ffa70d 100644 --- a/src/lib/protocols/ssl.c +++ b/src/lib/protocols/ssl.c @@ -626,6 +626,10 @@ void ndpi_search_ssl_tcp(struct ndpi_detection_module_struct *ndpi_struct, struc struct ndpi_packet_struct *packet = &flow->packet; u_int8_t ret; + if(flow->packet.tcp && (5222 == ntohs(flow->packet.tcp->dest))) + printf("%u - %u\n", ntohs(flow->packet.tcp->source), ntohs(flow->packet.tcp->dest)); + + if(packet->detected_protocol_stack[0] == NDPI_PROTOCOL_SSL) { if(flow->l4.tcp.ssl_stage == 3 && packet->payload_packet_len > 20 && flow->packet_counter < 5) { /* this should only happen, when we detected SSL with a packet that had parts of the certificate in subsequent packets diff --git a/src/lib/third_party/include/lruc.h b/src/lib/third_party/include/lruc.h deleted file mode 100644 index 55fb271fe..000000000 --- a/src/lib/third_party/include/lruc.h +++ /dev/null @@ -1,55 +0,0 @@ -#include -#include -#include - -#ifndef __lruc_header__ -#define __lruc_header__ - -// ------------------------------------------ -// errors -// ------------------------------------------ -typedef enum { - LRUC_NO_ERROR = 0, - LRUC_MISSING_CACHE, - LRUC_MISSING_KEY, - LRUC_MISSING_VALUE, - LRUC_PTHREAD_ERROR, - LRUC_VALUE_TOO_LARGE -} lruc_error; - - -// ------------------------------------------ -// types -// ------------------------------------------ -typedef struct { - void *value; - void *key; - uint32_t value_length; - uint32_t key_length; - uint64_t access_count; - void *next; -} lruc_item; - -typedef struct { - lruc_item **items; - uint64_t access_count; - uint64_t free_memory; - uint64_t total_memory; - uint64_t average_item_length; - uint32_t hash_table_size; - time_t seed; - lruc_item *free_items; - pthread_mutex_t *mutex; -} lruc; - - -// ------------------------------------------ -// api -// ------------------------------------------ -lruc *lruc_new(uint64_t cache_size, uint32_t average_length); -lruc_error lruc_free(lruc *cache); -lruc_error lruc_set(lruc *cache, void *key, uint32_t key_length, void *value, uint32_t value_length); -lruc_error lruc_get(lruc *cache, void *key, uint32_t key_length, void **value); -lruc_error lruc_delete(lruc *cache, void *key, uint32_t key_length); - -#endif diff --git a/src/lib/third_party/src/lruc.c b/src/lib/third_party/src/lruc.c deleted file mode 100644 index f08fb2ce1..000000000 --- a/src/lib/third_party/src/lruc.c +++ /dev/null @@ -1,294 +0,0 @@ -/* https://github.com/willcannings/C-LRU-Cache */ - -#include "lruc.h" -#include -#include -#include -#include - -// ------------------------------------------ -// private functions -// ------------------------------------------ -// MurmurHash2, by Austin Appleby -// http://sites.google.com/site/murmurhash/ -uint32_t lruc_hash(lruc *cache, void *key, uint32_t key_length) { - uint32_t m = 0x5bd1e995; - uint32_t r = 24; - uint32_t h = cache->seed ^ key_length; - char* data = (char *)key; - - while(key_length >= 4) { - uint32_t k = *(uint32_t *)data; - k *= m; - k ^= k >> r; - k *= m; - h *= m; - h ^= k; - data += 4; - key_length -= 4; - } - - switch(key_length) { - case 3: h ^= data[2] << 16; - case 2: h ^= data[1] << 8; - case 1: h ^= data[0]; - h *= m; - }; - - h ^= h >> 13; - h *= m; - h ^= h >> 15; - return h % cache->hash_table_size; -} - -// compare a key against an existing item's key -int lruc_cmp_keys(lruc_item *item, void *key, uint32_t key_length) { - if(key_length != item->key_length) - return 1; - else - return memcmp(key, item->key, key_length); -} - -// remove an item and push it to the free items queue -void lruc_remove_item(lruc *cache, lruc_item *prev, lruc_item *item, uint32_t hash_index) { - if(prev) - prev->next = item->next; - else - cache->items[hash_index] = (lruc_item *) item->next; - - // free memory and update the free memory counter - cache->free_memory += item->value_length; - free(item->value); - free(item->key); - - // push the item to the free items queue - memset(item, 0, sizeof(lruc_item)); - item->next = cache->free_items; - cache->free_items = item; -} - -// remove the least recently used item -// TODO: we can optimise this by finding the n lru items, where n = required_space / average_length -void lruc_remove_lru_item(lruc *cache) { - lruc_item *min_item = NULL, *min_prev = NULL; - lruc_item *item = NULL, *prev = NULL; - uint32_t i = 0, min_index = -1; - uint64_t min_access_count = -1; - - for(; i < cache->hash_table_size; i++) { - item = cache->items[i]; - prev = NULL; - - while(item) { - if(item->access_count < min_access_count || min_access_count == -1) { - min_access_count = item->access_count; - min_item = item; - min_prev = prev; - min_index = i; - } - prev = item; - item = item->next; - } - } - - if(min_item) - lruc_remove_item(cache, min_prev, min_item, min_index); -} - -// pop an existing item off the free queue, or create a new one -lruc_item *lruc_pop_or_create_item(lruc *cache) { - lruc_item *item = NULL; - - if(cache->free_items) { - item = cache->free_items; - cache->free_items = item->next; - } else { - item = (lruc_item *) calloc(sizeof(lruc_item), 1); - } - - return item; -} - -// error helpers -#define error_for(conditions, error) if(conditions) {return error;} -#define test_for_missing_cache() error_for(!cache, LRUC_MISSING_CACHE) -#define test_for_missing_key() error_for(!key || key_length == 0, LRUC_MISSING_KEY) -#define test_for_missing_value() error_for(!value || value_length == 0, LRUC_MISSING_VALUE) -#define test_for_value_too_large() error_for(value_length > cache->total_memory, LRUC_VALUE_TOO_LARGE) - -// lock helpers -#define lock_cache() if(pthread_mutex_lock(cache->mutex)) {\ - perror("LRU Cache unable to obtain mutex lock");\ - return LRUC_PTHREAD_ERROR;\ -} - -#define unlock_cache() if(pthread_mutex_unlock(cache->mutex)) {\ - perror("LRU Cache unable to release mutex lock");\ - return LRUC_PTHREAD_ERROR;\ -} - - -// ------------------------------------------ -// public api -// ------------------------------------------ -lruc *lruc_new(uint64_t cache_size, uint32_t average_length) { - // create the cache - lruc *cache = (lruc *) calloc(sizeof(lruc), 1); - if(!cache) { - perror("LRU Cache unable to create cache object"); - return NULL; - } - cache->hash_table_size = cache_size / average_length; - cache->average_item_length = average_length; - cache->free_memory = cache_size; - cache->total_memory = cache_size; - cache->seed = time(NULL); - - // size the hash table to a guestimate of the number of slots required (assuming a perfect hash) - cache->items = (lruc_item **) calloc(sizeof(lruc_item *), cache->hash_table_size); - if(!cache->items) { - perror("LRU Cache unable to create cache hash table"); - free(cache); - return NULL; - } - - // all cache calls are guarded by a mutex - cache->mutex = (pthread_mutex_t *) malloc(sizeof(pthread_mutex_t)); - if(pthread_mutex_init(cache->mutex, NULL)) { - perror("LRU Cache unable to initialise mutex"); - free(cache->items); - free(cache); - return NULL; - } - return cache; -} - - -lruc_error lruc_free(lruc *cache) { - test_for_missing_cache(); - - // free each of the cached items, and the hash table - lruc_item *item = NULL, *next = NULL; - uint32_t i = 0; - if(cache->items) { - for(; i < cache->hash_table_size; i++) { - item = cache->items[i]; - while(item) { - next = (lruc_item *) item->next; - free(item); - item = next; - } - } - free(cache->items); - } - - // free the cache - if(cache->mutex) { - if(pthread_mutex_destroy(cache->mutex)) { - perror("LRU Cache unable to destroy mutex"); - return LRUC_PTHREAD_ERROR; - } - } - free(cache); - - return LRUC_NO_ERROR; -} - - -lruc_error lruc_set(lruc *cache, void *key, uint32_t key_length, void *value, uint32_t value_length) { - test_for_missing_cache(); - test_for_missing_key(); - test_for_missing_value(); - test_for_value_too_large(); - lock_cache(); - - // see if the key already exists - uint32_t hash_index = lruc_hash(cache, key, key_length), required = 0; - lruc_item *item = NULL, *prev = NULL; - item = cache->items[hash_index]; - - while(item && lruc_cmp_keys(item, key, key_length)) { - prev = item; - item = (lruc_item *) item->next; - } - - if(item) { - // update the value and value_lengths - required = value_length - item->value_length; - free(item->value); - item->value = value; - item->value_length = value_length; - - } else { - // insert a new item - item = lruc_pop_or_create_item(cache); - item->value = value; - item->key = key; - item->value_length = value_length; - item->key_length = key_length; - required = value_length; - - if(prev) - prev->next = item; - else - cache->items[hash_index] = item; - } - item->access_count = ++cache->access_count; - - // remove as many items as necessary to free enough space - if(required > 0 && required > cache->free_memory) { - while(cache->free_memory < required) - lruc_remove_lru_item(cache); - } - cache->free_memory -= required; - unlock_cache(); - return LRUC_NO_ERROR; -} - - -lruc_error lruc_get(lruc *cache, void *key, uint32_t key_length, void **value) { - test_for_missing_cache(); - test_for_missing_key(); - lock_cache(); - - // loop until we find the item, or hit the end of a chain - uint32_t hash_index = lruc_hash(cache, key, key_length); - lruc_item *item = cache->items[hash_index]; - - while(item && lruc_cmp_keys(item, key, key_length)) - item = (lruc_item *) item->next; - - if(item) { - *value = item->value; - item->access_count = ++cache->access_count; - } else { - *value = NULL; - } - - unlock_cache(); - return LRUC_NO_ERROR; -} - - -lruc_error lruc_delete(lruc *cache, void *key, uint32_t key_length) { - test_for_missing_cache(); - test_for_missing_key(); - lock_cache(); - - // loop until we find the item, or hit the end of a chain - lruc_item *item = NULL, *prev = NULL; - uint32_t hash_index = lruc_hash(cache, key, key_length); - item = cache->items[hash_index]; - - while(item && lruc_cmp_keys(item, key, key_length)) { - prev = item; - item = (lruc_item *) item->next; - } - - if(item) { - lruc_remove_item(cache, prev, item, hash_index); - } - - unlock_cache(); - return LRUC_NO_ERROR; -} -- cgit v1.2.3 From e2ffcd722958857df232d8a02a9809b05d861049 Mon Sep 17 00:00:00 2001 From: Luca Deri Date: Wed, 23 Jan 2019 00:53:56 +0100 Subject: Improved HTTP response code handling --- src/include/ndpi_typedefs.h | 2 +- src/lib/ndpi_main.c | 18 ++++++++++++------ src/lib/protocols/http.c | 22 +++++++++++++++++++--- 3 files changed, 32 insertions(+), 10 deletions(-) (limited to 'src/lib/protocols/http.c') diff --git a/src/include/ndpi_typedefs.h b/src/include/ndpi_typedefs.h index cf1af8bc3..172763789 100644 --- a/src/include/ndpi_typedefs.h +++ b/src/include/ndpi_typedefs.h @@ -1071,7 +1071,7 @@ struct ndpi_flow_struct { char *url, *content_type; u_int8_t num_request_headers, num_response_headers; u_int8_t request_version; /* 0=1.0 and 1=1.1. Create an enum for this? */ - u_char response_status_code[5]; /* 200, 404, etc. */ + u_int16_t response_status_code; /* 200, 404, etc. */ } http; union { diff --git a/src/lib/ndpi_main.c b/src/lib/ndpi_main.c index fcf86449a..0b7af360d 100644 --- a/src/lib/ndpi_main.c +++ b/src/lib/ndpi_main.c @@ -4792,12 +4792,18 @@ void ndpi_parse_packet_line_info(struct ndpi_detection_module_struct *ndpi_struc packet->http_num_headers++; /* Set server HTTP response code */ - strncpy((char*)flow->http.response_status_code, (char*)packet->http_response.ptr, 3); - flow->http.response_status_code[4] = '\0'; - - NDPI_LOG_DBG2(ndpi_struct, - "ndpi_parse_packet_line_info: HTTP response parsed: \"%.*s\"\n", - packet->http_response.len, packet->http_response.ptr); + if(packet->payload_packet_len >= 12) { + char buf[4]; + + /* Set server HTTP response code */ + strncpy(buf, (char*)&packet->payload[9], 3); + buf[3] = '\0'; + + flow->http.response_status_code = atoi(buf); + /* https://en.wikipedia.org/wiki/List_of_HTTP_status_codes */ + if((flow->http.response_status_code < 100) || (flow->http.response_status_code > 509)) + flow->http.response_status_code = 0; /* Out of range */ + } } /* "Server:" header line in HTTP response */ diff --git a/src/lib/protocols/http.c b/src/lib/protocols/http.c index 16b122d06..4ce80f9c9 100644 --- a/src/lib/protocols/http.c +++ b/src/lib/protocols/http.c @@ -26,6 +26,7 @@ #define NDPI_CURRENT_PROTO NDPI_PROTOCOL_HTTP #include "ndpi_api.h" +#include static void ndpi_int_http_add_connection(struct ndpi_detection_module_struct *ndpi_struct, struct ndpi_flow_struct *flow, @@ -54,7 +55,7 @@ static void ndpi_int_http_add_connection(struct ndpi_detection_module_struct *nd flow->packet.detected_protocol_stack[0] = NDPI_PROTOCOL_UNKNOWN, flow->packet.detected_protocol_stack[1] = NDPI_PROTOCOL_UNKNOWN; } } else { - if((!ndpi_struct->http_dont_dissect_response) && (flow->http.response_status_code[0] == '\0')) { + if((!ndpi_struct->http_dont_dissect_response) && (flow->http.response_status_code == 0)) { flow->http_upper_protocol = flow->detected_protocol_stack[0], flow->http_lower_protocol = flow->detected_protocol_stack[1]; flow->detected_protocol_stack[0] = NDPI_PROTOCOL_UNKNOWN, flow->detected_protocol_stack[1] = NDPI_PROTOCOL_UNKNOWN; flow->packet.detected_protocol_stack[0] = NDPI_PROTOCOL_UNKNOWN, flow->packet.detected_protocol_stack[1] = NDPI_PROTOCOL_UNKNOWN; @@ -211,10 +212,11 @@ static void check_content_type_and_change_protocol(struct ndpi_detection_module_ struct ndpi_packet_struct *packet = &flow->packet; u_int8_t a; - if((!ndpi_struct->http_dont_dissect_response) && flow->http_detected && (flow->http.response_status_code[0] != 0)) { + if((!ndpi_struct->http_dont_dissect_response) && flow->http_detected && (flow->http.response_status_code != 0)) { ndpi_set_detected_protocol(ndpi_struct, flow, flow->http_upper_protocol, flow->http_lower_protocol); #ifdef DEBUG - printf("[%s] [http_dont_dissect_response: %u]->> %s\n", __FUNCTION__, ndpi_struct->http_dont_dissect_response, flow->http.response_status_code); + printf("[%s] [http_dont_dissect_response: %u]->> %s\n", + __FUNCTION__, ndpi_struct->http_dont_dissect_response, flow->http.response_status_code); #endif return; } @@ -600,6 +602,20 @@ static void ndpi_check_http_tcp(struct ndpi_detection_module_struct *ndpi_struct if(packet->payload_packet_len >= 7 && memcmp(packet->payload, "HTTP/1.", 7) == 0) { NDPI_LOG_INFO(ndpi_struct, "found HTTP response\n"); + + if(packet->payload_packet_len >= 12) { + char buf[4]; + + /* Set server HTTP response code */ + strncpy(buf, (char*)&packet->payload[9], 3); + buf[3] = '\0'; + + flow->http.response_status_code = atoi(buf); + /* https://en.wikipedia.org/wiki/List_of_HTTP_status_codes */ + if((flow->http.response_status_code < 100) || (flow->http.response_status_code > 509)) + flow->http.response_status_code = 0; /* Out of range */ + } + ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_HTTP); check_content_type_and_change_protocol(ndpi_struct, flow); return; -- cgit v1.2.3 From 89d548f9d30a006caaf304a962f4cc4cf8cdf6c1 Mon Sep 17 00:00:00 2001 From: Luca Deri Date: Sun, 10 Mar 2019 21:18:19 +0100 Subject: Removed categories based on file content --- src/include/ndpi_typedefs.h | 10 -- src/lib/ndpi_content_match.c.inc | 57 --------- src/lib/ndpi_main.c | 6 - src/lib/protocols/http.c | 257 +-------------------------------------- 4 files changed, 2 insertions(+), 328 deletions(-) (limited to 'src/lib/protocols/http.c') diff --git a/src/include/ndpi_typedefs.h b/src/include/ndpi_typedefs.h index 917aa6827..39995d662 100644 --- a/src/include/ndpi_typedefs.h +++ b/src/include/ndpi_typedefs.h @@ -791,16 +791,6 @@ typedef enum { NDPI_PROTOCOL_CATEGORY_CUSTOM_4, /* User custom category 4 */ NDPI_PROTOCOL_CATEGORY_CUSTOM_5, /* User custom category 5 */ - /* Payload Content */ - NDPI_CONTENT_CATEGORY_AVI, - NDPI_CONTENT_CATEGORY_FLASH, - NDPI_CONTENT_CATEGORY_OGG, - NDPI_CONTENT_CATEGORY_MPEG, - NDPI_CONTENT_CATEGORY_QUICKTIME, - NDPI_CONTENT_CATEGORY_REALMEDIA, - NDPI_CONTENT_CATEGORY_WINDOWSMEDIA, - NDPI_CONTENT_CATEGORY_WEBM, - /* Further categories... */ NDPI_PROTOCOL_CATEGORY_MUSIC, NDPI_PROTOCOL_CATEGORY_VIDEO, diff --git a/src/lib/ndpi_content_match.c.inc b/src/lib/ndpi_content_match.c.inc index c71913081..3261618b9 100644 --- a/src/lib/ndpi_content_match.c.inc +++ b/src/lib/ndpi_content_match.c.inc @@ -8418,63 +8418,6 @@ ndpi_protocol_match host_match[] = { { NULL, NULL, NULL, 0 } }; - -/* - Mime-type content match match -*/ -ndpi_protocol_match content_match[] = { - { "audio/mpeg", NULL, NULL, NULL, NDPI_CONTENT_CATEGORY_MPEG, NDPI_PROTOCOL_CATEGORY_MEDIA, NDPI_PROTOCOL_FUN }, - { "audio/x-mpeg", NULL, NULL, NULL, NDPI_CONTENT_CATEGORY_MPEG, NDPI_PROTOCOL_CATEGORY_MEDIA, NDPI_PROTOCOL_FUN }, - { "audio/mpeg3", NULL, NULL, NULL, NDPI_CONTENT_CATEGORY_MPEG, NDPI_PROTOCOL_CATEGORY_MEDIA, NDPI_PROTOCOL_FUN }, - { "audio/mp4a", NULL, NULL, NULL, NDPI_CONTENT_CATEGORY_MPEG, NDPI_PROTOCOL_CATEGORY_MEDIA, NDPI_PROTOCOL_FUN }, - { "video/mpeg", NULL, NULL, NULL, NDPI_CONTENT_CATEGORY_MPEG, NDPI_PROTOCOL_CATEGORY_MEDIA, NDPI_PROTOCOL_FUN }, - { "video/nsv", NULL, NULL, NULL, NDPI_CONTENT_CATEGORY_MPEG, NDPI_PROTOCOL_CATEGORY_MEDIA, NDPI_PROTOCOL_FUN }, - { "misc/ultravox", NULL, NULL, NULL, NDPI_CONTENT_CATEGORY_MPEG, NDPI_PROTOCOL_CATEGORY_MEDIA, NDPI_PROTOCOL_FUN }, - { "audio/ogg", NULL, NULL, NULL, NDPI_CONTENT_CATEGORY_OGG, NDPI_PROTOCOL_CATEGORY_MEDIA, NDPI_PROTOCOL_FUN }, - { "video/ogg", NULL, NULL, NULL, NDPI_CONTENT_CATEGORY_OGG, NDPI_PROTOCOL_CATEGORY_MEDIA, NDPI_PROTOCOL_FUN }, - { "application/ogg", NULL, NULL, NULL, NDPI_CONTENT_CATEGORY_OGG, NDPI_PROTOCOL_CATEGORY_MEDIA, NDPI_PROTOCOL_FUN }, - { ".adobe.", NULL, NULL, NULL, NDPI_CONTENT_CATEGORY_FLASH, NDPI_PROTOCOL_CATEGORY_MEDIA, NDPI_PROTOCOL_FUN }, - { "video/flv", NULL, NULL, NULL, NDPI_CONTENT_CATEGORY_FLASH, NDPI_PROTOCOL_CATEGORY_MEDIA, NDPI_PROTOCOL_FUN }, - { "video/x-flv", NULL, NULL, NULL, NDPI_CONTENT_CATEGORY_FLASH, NDPI_PROTOCOL_CATEGORY_MEDIA, NDPI_PROTOCOL_FUN }, - { "application/x-fcs", NULL, NULL, NULL, NDPI_CONTENT_CATEGORY_FLASH, NDPI_PROTOCOL_CATEGORY_MEDIA, NDPI_PROTOCOL_FUN }, - { "application/x-shockwave-flash",NULL, NULL, NULL, NDPI_CONTENT_CATEGORY_FLASH, NDPI_PROTOCOL_CATEGORY_MEDIA, NDPI_PROTOCOL_ACCEPTABLE }, - { "video/flash", NULL, NULL, NULL, NDPI_CONTENT_CATEGORY_FLASH, NDPI_PROTOCOL_CATEGORY_MEDIA, NDPI_PROTOCOL_FUN }, - { "application/flv", NULL, NULL, NULL, NDPI_CONTENT_CATEGORY_FLASH, NDPI_PROTOCOL_CATEGORY_MEDIA, NDPI_PROTOCOL_FUN }, - { "flv-application/octet-stream", NULL, NULL, NULL, NDPI_CONTENT_CATEGORY_FLASH, NDPI_PROTOCOL_CATEGORY_MEDIA, NDPI_PROTOCOL_FUN }, - { "application/futuresplash", NULL, NULL, NULL, NDPI_CONTENT_CATEGORY_FLASH, NDPI_PROTOCOL_CATEGORY_MEDIA, NDPI_PROTOCOL_FUN }, - { "video/quicktime", NULL, NULL, NULL, NDPI_CONTENT_CATEGORY_QUICKTIME, NDPI_PROTOCOL_CATEGORY_MEDIA, NDPI_PROTOCOL_FUN }, - { "video/mp4", NULL, NULL, NULL, NDPI_CONTENT_CATEGORY_QUICKTIME, NDPI_PROTOCOL_CATEGORY_MEDIA, NDPI_PROTOCOL_FUN }, - { "video/x-m4v", NULL, NULL, NULL, NDPI_CONTENT_CATEGORY_QUICKTIME, NDPI_PROTOCOL_CATEGORY_MEDIA, NDPI_PROTOCOL_FUN }, - { "audio/x-pn-realaudio", NULL, NULL, NULL, NDPI_CONTENT_CATEGORY_REALMEDIA, NDPI_PROTOCOL_CATEGORY_MEDIA, NDPI_PROTOCOL_FUN }, - { "application/vnd.rn-realmedia", NULL, NULL, NULL, NDPI_CONTENT_CATEGORY_REALMEDIA, NDPI_PROTOCOL_CATEGORY_MEDIA, NDPI_PROTOCOL_FUN }, - { "video/x-ms-", NULL, NULL, NULL, NDPI_CONTENT_CATEGORY_WINDOWSMEDIA, NDPI_PROTOCOL_CATEGORY_MEDIA, NDPI_PROTOCOL_FUN }, - { "asf", NULL, NULL, NULL, NDPI_CONTENT_CATEGORY_WINDOWSMEDIA, NDPI_PROTOCOL_CATEGORY_MEDIA, NDPI_PROTOCOL_FUN }, - { "asx", NULL, NULL, NULL, NDPI_CONTENT_CATEGORY_WINDOWSMEDIA, NDPI_PROTOCOL_CATEGORY_MEDIA, NDPI_PROTOCOL_FUN }, - { "video/x-msvideo", NULL, NULL, NULL, NDPI_CONTENT_CATEGORY_WINDOWSMEDIA, NDPI_PROTOCOL_CATEGORY_MEDIA, NDPI_PROTOCOL_FUN }, - { "audio/x-wav", NULL, NULL, NULL, NDPI_CONTENT_CATEGORY_WINDOWSMEDIA, NDPI_PROTOCOL_CATEGORY_MEDIA, NDPI_PROTOCOL_FUN }, - { "application/vnd.ms.wms-hdr.asfv1", NULL, NULL, NULL, NDPI_CONTENT_CATEGORY_WINDOWSMEDIA, NDPI_PROTOCOL_CATEGORY_MEDIA, NDPI_PROTOCOL_FUN }, - { "NSPlayer/", NULL, NULL, NULL, NDPI_CONTENT_CATEGORY_WINDOWSMEDIA, NDPI_PROTOCOL_CATEGORY_MEDIA, NDPI_PROTOCOL_FUN }, - { "Xbox Live Client/", NULL, NULL, NULL, NDPI_PROTOCOL_XBOX, NDPI_PROTOCOL_CATEGORY_GAME, NDPI_PROTOCOL_FUN }, - { "Windows-Update-Agent", NULL, NULL, NULL, NDPI_PROTOCOL_WINDOWS_UPDATE, NDPI_PROTOCOL_CATEGORY_MEDIA, NDPI_PROTOCOL_ACCEPTABLE }, - { "audio/webm", NULL, NULL, NULL, NDPI_CONTENT_CATEGORY_WEBM, NDPI_PROTOCOL_CATEGORY_MEDIA, NDPI_PROTOCOL_FUN }, - { "video/webm", NULL, NULL, NULL, NDPI_CONTENT_CATEGORY_WEBM, NDPI_PROTOCOL_CATEGORY_MEDIA, NDPI_PROTOCOL_FUN }, - { "application/x-rtsp-tunnelled", NULL, NULL, NULL, NDPI_PROTOCOL_RTSP, NDPI_PROTOCOL_CATEGORY_MEDIA, NDPI_PROTOCOL_FUN }, - { "application/vnd.apple.mpegurl",NULL, NULL, NULL, NDPI_CONTENT_CATEGORY_MPEG, NDPI_PROTOCOL_CATEGORY_MEDIA, NDPI_PROTOCOL_FUN }, - { "application/x-tar", NULL, NULL, NULL, NDPI_PROTOCOL_HTTP_DOWNLOAD, NDPI_PROTOCOL_CATEGORY_DOWNLOAD_FT, NDPI_PROTOCOL_ACCEPTABLE }, - { "application/octet-stream", NULL, NULL, NULL, NDPI_PROTOCOL_HTTP_DOWNLOAD, NDPI_PROTOCOL_CATEGORY_DOWNLOAD_FT, NDPI_PROTOCOL_ACCEPTABLE }, - { "application/mac-binary", NULL, NULL, NULL, NDPI_PROTOCOL_HTTP_DOWNLOAD, NDPI_PROTOCOL_CATEGORY_DOWNLOAD_FT, NDPI_PROTOCOL_ACCEPTABLE }, - { "/x-bzip", NULL, NULL, NULL, NDPI_PROTOCOL_HTTP_DOWNLOAD, NDPI_PROTOCOL_CATEGORY_DOWNLOAD_FT, NDPI_PROTOCOL_ACCEPTABLE }, - { "/x-gzip", NULL, NULL, NULL, NDPI_PROTOCOL_HTTP_DOWNLOAD, NDPI_PROTOCOL_CATEGORY_DOWNLOAD_FT, NDPI_PROTOCOL_ACCEPTABLE }, - { "/x-zip", NULL, NULL, NULL, NDPI_PROTOCOL_HTTP_DOWNLOAD, NDPI_PROTOCOL_CATEGORY_DOWNLOAD_FT, NDPI_PROTOCOL_ACCEPTABLE }, - { "/zip", NULL, NULL, NULL, NDPI_PROTOCOL_HTTP_DOWNLOAD, NDPI_PROTOCOL_CATEGORY_DOWNLOAD_FT, NDPI_PROTOCOL_ACCEPTABLE }, - { "binhex", NULL, NULL, NULL, NDPI_PROTOCOL_HTTP_DOWNLOAD, NDPI_PROTOCOL_CATEGORY_DOWNLOAD_FT, NDPI_PROTOCOL_ACCEPTABLE }, - { "/base64", NULL, NULL, NULL, NDPI_PROTOCOL_HTTP_DOWNLOAD, NDPI_PROTOCOL_CATEGORY_DOWNLOAD_FT, NDPI_PROTOCOL_ACCEPTABLE }, - { "application/gnutar", NULL, NULL, NULL, NDPI_PROTOCOL_HTTP_DOWNLOAD, NDPI_PROTOCOL_CATEGORY_DOWNLOAD_FT, NDPI_PROTOCOL_ACCEPTABLE }, - { "application/x-compressed", NULL, NULL, NULL, NDPI_PROTOCOL_HTTP_DOWNLOAD, NDPI_PROTOCOL_CATEGORY_DOWNLOAD_FT, NDPI_PROTOCOL_ACCEPTABLE }, - - { NULL, NULL, NULL, 0 } -}; - /* ****************************************************** */ /* diff --git a/src/lib/ndpi_main.c b/src/lib/ndpi_main.c index b8cdf410b..b0f1383d3 100644 --- a/src/lib/ndpi_main.c +++ b/src/lib/ndpi_main.c @@ -915,12 +915,6 @@ static void init_string_based_protocols(struct ndpi_detection_module_struct *ndp // ac_automata_display(ndpi_mod->host_automa.ac_automa, 'n'); #endif - for(i=0; content_match[i].string_to_match != NULL; i++) - ndpi_add_content_subprotocol(ndpi_mod, content_match[i].string_to_match, - content_match[i].protocol_id, - content_match[i].protocol_category, - content_match[i].protocol_breed); - for(i=0; ndpi_en_bigrams[i] != NULL; i++) ndpi_string_to_automa(ndpi_mod, &ndpi_mod->bigrams_automa, (char*)ndpi_en_bigrams[i], diff --git a/src/lib/protocols/http.c b/src/lib/protocols/http.c index 4ce80f9c9..37f23e26a 100644 --- a/src/lib/protocols/http.c +++ b/src/lib/protocols/http.c @@ -65,93 +65,6 @@ static void ndpi_int_http_add_connection(struct ndpi_detection_module_struct *nd flow->http_detected = 1, flow->guessed_category = category; } -#ifdef NDPI_CONTENT_CATEGORY_FLASH -static void flash_check_http_payload(struct ndpi_detection_module_struct - *ndpi_struct, struct ndpi_flow_struct *flow) -{ - struct ndpi_packet_struct *packet = &flow->packet; - const u_int8_t *pos; - - if(packet->empty_line_position_set == 0 || (packet->empty_line_position + 10) > (packet->payload_packet_len)) - return; - - pos = &packet->payload[packet->empty_line_position] + 2; - - if(memcmp(pos, "FLV", 3) == 0 && pos[3] == 0x01 && (pos[4] == 0x01 || pos[4] == 0x04 || pos[4] == 0x05) - && pos[5] == 0x00 && pos[6] == 0x00 && pos[7] == 0x00 && pos[8] == 0x09) { - - NDPI_LOG_INFO(ndpi_struct, "found Flash content in HTTP\n"); - ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_CONTENT_CATEGORY_FLASH); - } -} -#endif - -#ifdef NDPI_CONTENT_CATEGORY_AVI -static void avi_check_http_payload(struct ndpi_detection_module_struct *ndpi_struct, struct ndpi_flow_struct *flow) -{ - struct ndpi_packet_struct *packet = &flow->packet; - - - NDPI_LOG_DBG2(ndpi_struct, "called avi_check_http_payload: %u %u %u\n", - packet->empty_line_position_set, flow->l4.tcp.http_empty_line_seen, packet->empty_line_position); - - if(packet->empty_line_position_set == 0 && flow->l4.tcp.http_empty_line_seen == 0) - return; - - if(packet->empty_line_position_set != 0 && ((packet->empty_line_position + 20) > (packet->payload_packet_len)) - && flow->l4.tcp.http_empty_line_seen == 0) { - flow->l4.tcp.http_empty_line_seen = 1; - return; - } - - if(flow->l4.tcp.http_empty_line_seen == 1) { - if(packet->payload_packet_len > 20 && memcmp(packet->payload, "RIFF", 4) == 0 - && memcmp(packet->payload + 8, "AVI LIST", 8) == 0) { - NDPI_LOG_INFO(ndpi_struct, "found Avi content in HTTP\n"); - ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_CONTENT_CATEGORY_AVI); - } - flow->l4.tcp.http_empty_line_seen = 0; - return; - } - - /** - for reference see http://msdn.microsoft.com/archive/default.asp?url=/archive/en-us/directx9_c/directx/htm/avirifffilereference.asp - **/ - if(packet->empty_line_position_set != 0) { - - u_int32_t p = packet->empty_line_position + 2; - - // check for avi header - NDPI_LOG_DBG2(ndpi_struct, "p = %u\n", p); - - if((p + 16) <= packet->payload_packet_len && memcmp(&packet->payload[p], "RIFF", 4) == 0 - && memcmp(&packet->payload[p + 8], "AVI LIST", 8) == 0) { - NDPI_LOG_INFO(ndpi_struct, "found Avi content in HTTP\n"); - ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_CONTENT_CATEGORY_AVI); - } - } -} -#endif - -static void teamviewer_check_http_payload(struct ndpi_detection_module_struct *ndpi_struct, struct ndpi_flow_struct *flow) -{ - struct ndpi_packet_struct *packet = &flow->packet; - const u_int8_t *pos; - - NDPI_LOG_DBG2(ndpi_struct, "called teamviewer_check_http_payload: %u %u %u\n", - packet->empty_line_position_set, flow->l4.tcp.http_empty_line_seen, packet->empty_line_position); - - if(packet->empty_line_position_set == 0 || (packet->empty_line_position + 5) > (packet->payload_packet_len)) - return; - - pos = &packet->payload[packet->empty_line_position] + 2; - - if(pos[0] == 0x17 && pos[1] == 0x24) { - NDPI_LOG_INFO(ndpi_struct, "found TeamViewer content in HTTP\n"); - ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_TEAMVIEWER); - } -} - static void rtsp_parse_packet_acceptline(struct ndpi_detection_module_struct *ndpi_struct, struct ndpi_flow_struct *flow) { @@ -461,15 +374,6 @@ static void check_content_type_and_change_protocol(struct ndpi_detection_module_ } } - /* search for line startin with "Icy-MetaData" */ - for (a = 0; a < packet->parsed_lines; a++) { - if(packet->line[a].len > 11 && memcmp(packet->line[a].ptr, "Icy-MetaData", 12) == 0) { - NDPI_LOG_INFO(ndpi_struct, "found MPEG: Icy-MetaData\n"); - ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_CONTENT_CATEGORY_MPEG); - return; - } - } - if(packet->content_line.ptr != NULL && packet->content_line.len != 0) { NDPI_LOG_DBG2(ndpi_struct, "Content Type line found %.*s\n", packet->content_line.len, packet->content_line.ptr); @@ -486,20 +390,8 @@ static void check_content_type_and_change_protocol(struct ndpi_detection_module_ ndpi_int_http_add_connection(ndpi_struct, flow, packet->detected_protocol_stack[0]); } -static void check_http_payload(struct ndpi_detection_module_struct *ndpi_struct, struct ndpi_flow_struct *flow) -{ - NDPI_LOG_DBG2(ndpi_struct, "called check_http_payload\n"); - -#ifdef NDPI_CONTENT_CATEGORY_FLASH - if(NDPI_COMPARE_PROTOCOL_TO_BITMASK(ndpi_struct->detection_bitmask, NDPI_CONTENT_CATEGORY_FLASH) != 0) - flash_check_http_payload(ndpi_struct, flow); -#endif -#ifdef NDPI_CONTENT_CATEGORY_AVI - if(NDPI_COMPARE_PROTOCOL_TO_BITMASK(ndpi_struct->detection_bitmask, NDPI_CONTENT_CATEGORY_AVI) != 0) - avi_check_http_payload(ndpi_struct, flow); -#endif - - teamviewer_check_http_payload(ndpi_struct, flow); +static void check_http_payload(struct ndpi_detection_module_struct *ndpi_struct, struct ndpi_flow_struct *flow) { + /* Add here your paylod code check */ } /** @@ -557,25 +449,6 @@ static u_int16_t http_request_url_offset(struct ndpi_detection_module_struct *nd static void http_bitmask_exclude_other(struct ndpi_flow_struct *flow) { -#ifdef NDPI_CONTENT_CATEGORY_MPEG - NDPI_ADD_PROTOCOL_TO_BITMASK(flow->excluded_protocol_bitmask, NDPI_CONTENT_CATEGORY_MPEG); -#endif -#ifdef NDPI_CONTENT_CATEGORY_QUICKTIME - NDPI_ADD_PROTOCOL_TO_BITMASK(flow->excluded_protocol_bitmask, NDPI_CONTENT_CATEGORY_QUICKTIME); -#endif -#ifdef NDPI_CONTENT_CATEGORY_WINDOWSMEDIA - NDPI_ADD_PROTOCOL_TO_BITMASK(flow->excluded_protocol_bitmask, NDPI_CONTENT_CATEGORY_WINDOWSMEDIA); -#endif -#ifdef NDPI_CONTENT_CATEGORY_REALMEDIA - NDPI_ADD_PROTOCOL_TO_BITMASK(flow->excluded_protocol_bitmask, NDPI_CONTENT_CATEGORY_REALMEDIA); -#endif -#ifdef NDPI_CONTENT_CATEGORY_AVI - NDPI_ADD_PROTOCOL_TO_BITMASK(flow->excluded_protocol_bitmask, NDPI_CONTENT_CATEGORY_AVI); -#endif -#ifdef NDPI_CONTENT_CATEGORY_OGG - NDPI_ADD_PROTOCOL_TO_BITMASK(flow->excluded_protocol_bitmask, NDPI_CONTENT_CATEGORY_OGG); -#endif - NDPI_ADD_PROTOCOL_TO_BITMASK(flow->excluded_protocol_bitmask, NDPI_PROTOCOL_XBOX); } @@ -985,130 +858,4 @@ void init_http_dissector(struct ndpi_detection_module_struct *ndpi_struct, u_int SAVE_DETECTION_BITMASK_AS_UNKNOWN, ADD_TO_DETECTION_BITMASK); *id += 1; - -#if 0 - ndpi_set_bitmask_protocol_detection("HTTP_Proxy", ndpi_struct, detection_bitmask, *id, - NDPI_PROTOCOL_HTTP_PROXY, - ndpi_search_http_tcp, - NDPI_SELECTION_BITMASK_PROTOCOL_V4_V6_TCP_WITH_PAYLOAD, - SAVE_DETECTION_BITMASK_AS_UNKNOWN, - ADD_TO_DETECTION_BITMASK); - *id += 1; - -#ifdef NDPI_CONTENT_CATEGORY_MPEG - ndpi_set_bitmask_protocol_detection("MPEG", ndpi_struct, detection_bitmask, *id, - NDPI_CONTENT_CATEGORY_MPEG, - ndpi_search_http_tcp, - NDPI_SELECTION_BITMASK_PROTOCOL_V4_V6_TCP_WITH_PAYLOAD, - NO_SAVE_DETECTION_BITMASK_AS_UNKNOWN, - ADD_TO_DETECTION_BITMASK); - - *id += 1; -#endif -#ifdef NDPI_CONTENT_CATEGORY_FLASH - ndpi_set_bitmask_protocol_detection("Flash", ndpi_struct, detection_bitmask, *id, - NDPI_CONTENT_CATEGORY_FLASH, - ndpi_search_http_tcp, - NDPI_SELECTION_BITMASK_PROTOCOL_V4_V6_TCP_WITH_PAYLOAD, - NO_SAVE_DETECTION_BITMASK_AS_UNKNOWN, - ADD_TO_DETECTION_BITMASK); - *id += 1; -#endif -#ifdef NDPI_CONTENT_CATEGORY_QUICKTIME - ndpi_set_bitmask_protocol_detection("QuickTime", ndpi_struct, detection_bitmask, *id, - NDPI_CONTENT_CATEGORY_QUICKTIME, - ndpi_search_http_tcp, - NDPI_SELECTION_BITMASK_PROTOCOL_V4_V6_TCP_WITH_PAYLOAD, - NO_SAVE_DETECTION_BITMASK_AS_UNKNOWN, - ADD_TO_DETECTION_BITMASK); - *id += 1; -#endif -#ifdef NDPI_CONTENT_CATEGORY_REALMEDIA - ndpi_set_bitmask_protocol_detection("RealMedia", ndpi_struct, detection_bitmask, *id, - NDPI_CONTENT_CATEGORY_REALMEDIA, - ndpi_search_http_tcp, - NDPI_SELECTION_BITMASK_PROTOCOL_V4_V6_TCP_WITH_PAYLOAD, - NO_SAVE_DETECTION_BITMASK_AS_UNKNOWN, - ADD_TO_DETECTION_BITMASK); - *id += 1; -#endif -#ifdef NDPI_CONTENT_CATEGORY_WINDOWSMEDIA - ndpi_set_bitmask_protocol_detection("WindowsMedia", ndpi_struct, detection_bitmask, *id, - NDPI_CONTENT_CATEGORY_WINDOWSMEDIA, - ndpi_search_http_tcp, - NDPI_SELECTION_BITMASK_PROTOCOL_V4_V6_TCP_WITH_PAYLOAD, - NO_SAVE_DETECTION_BITMASK_AS_UNKNOWN, - ADD_TO_DETECTION_BITMASK); - *id += 1; -#endif -#ifdef NDPI_CONTENT_CATEGORY_MMS - ndpi_set_bitmask_protocol_detection("MMS", ndpi_struct, detection_bitmask, *id, - NDPI_CONTENT_CATEGORY_MMS, - ndpi_search_http_tcp, - NDPI_SELECTION_BITMASK_PROTOCOL_V4_V6_TCP_WITH_PAYLOAD, - NO_SAVE_DETECTION_BITMASK_AS_UNKNOWN, - ADD_TO_DETECTION_BITMASK); - *id += 1; -#endif - - ndpi_set_bitmask_protocol_detection("Xbox", ndpi_struct, detection_bitmask, *id, - NDPI_PROTOCOL_XBOX, - ndpi_search_http_tcp, - NDPI_SELECTION_BITMASK_PROTOCOL_V4_V6_TCP_WITH_PAYLOAD, - NO_SAVE_DETECTION_BITMASK_AS_UNKNOWN, - ADD_TO_DETECTION_BITMASK); - *id += 1; - - ndpi_set_bitmask_protocol_detection("QQ", ndpi_struct, detection_bitmask, *id, - NDPI_PROTOCOL_QQ, - ndpi_search_http_tcp, - NDPI_SELECTION_BITMASK_PROTOCOL_V4_V6_TCP_WITH_PAYLOAD, - NO_SAVE_DETECTION_BITMASK_AS_UNKNOWN, - ADD_TO_DETECTION_BITMASK); - *id += 1; - -#ifdef NDPI_CONTENT_CATEGORY_AVI - ndpi_set_bitmask_protocol_detection("AVI", ndpi_struct, detection_bitmask, *id, - NDPI_CONTENT_CATEGORY_AVI, - ndpi_search_http_tcp, - NDPI_SELECTION_BITMASK_PROTOCOL_V4_V6_TCP_WITH_PAYLOAD, - NO_SAVE_DETECTION_BITMASK_AS_UNKNOWN, - ADD_TO_DETECTION_BITMASK); - *id += 1; -#endif -#ifdef NDPI_CONTENT_CATEGORY_OGG - ndpi_set_bitmask_protocol_detection("OggVorbis", ndpi_struct, detection_bitmask, *id, - NDPI_CONTENT_CATEGORY_OGG, - ndpi_search_http_tcp, - NDPI_SELECTION_BITMASK_PROTOCOL_V4_V6_TCP_WITH_PAYLOAD, - NO_SAVE_DETECTION_BITMASK_AS_UNKNOWN, - ADD_TO_DETECTION_BITMASK); - *id += 1; -#endif - - /* Update excluded protocol bitmask */ - NDPI_BITMASK_SET(ndpi_struct->callback_buffer[a].excluded_protocol_bitmask, - ndpi_struct->callback_buffer[a].detection_bitmask); - - /*Delete protocol from excluded protocol bitmask*/ - NDPI_DEL_PROTOCOL_FROM_BITMASK(ndpi_struct->callback_buffer[a].excluded_protocol_bitmask, NDPI_PROTOCOL_UNKNOWN); - - NDPI_DEL_PROTOCOL_FROM_BITMASK(ndpi_struct->callback_buffer[a].excluded_protocol_bitmask, NDPI_PROTOCOL_QQ); - -#ifdef NDPI_CONTENT_CATEGORY_FLASH - NDPI_DEL_PROTOCOL_FROM_BITMASK(ndpi_struct->callback_buffer[a].excluded_protocol_bitmask, NDPI_CONTENT_CATEGORY_FLASH); -#endif - - NDPI_DEL_PROTOCOL_FROM_BITMASK(ndpi_struct->callback_buffer[a].excluded_protocol_bitmask, NDPI_CONTENT_CATEGORY_MMS); - - NDPI_DEL_PROTOCOL_FROM_BITMASK(ndpi_struct->callback_buffer[a].excluded_protocol_bitmask, NDPI_PROTOCOL_XBOX); - - NDPI_BITMASK_SET(ndpi_struct->generic_http_packet_bitmask, ndpi_struct->callback_buffer[a].detection_bitmask); - - NDPI_DEL_PROTOCOL_FROM_BITMASK(ndpi_struct->generic_http_packet_bitmask, NDPI_PROTOCOL_UNKNOWN); - - /* Update callback_buffer index */ - a++; - -#endif } -- cgit v1.2.3 From b1bd7d6f8c7a18beabaf8224056f13c2193dce03 Mon Sep 17 00:00:00 2001 From: chiehminw Date: Tue, 12 Mar 2019 16:11:33 +0800 Subject: Add NDPI prefix for HTTP_METHOD enum to avoid name collisions ref: https://stackoverflow.com/questions/35380279/avoid-name-collisions-with-enum-in-c-c99 Signed-off-by: chiehminw --- src/include/ndpi_typedefs.h | 18 +++++++++--------- src/lib/protocols/http.c | 22 +++++++++++----------- 2 files changed, 20 insertions(+), 20 deletions(-) (limited to 'src/lib/protocols/http.c') diff --git a/src/include/ndpi_typedefs.h b/src/include/ndpi_typedefs.h index a5f1864e0..e561deb47 100644 --- a/src/include/ndpi_typedefs.h +++ b/src/include/ndpi_typedefs.h @@ -356,15 +356,15 @@ PACK_ON struct tinc_cache_entry { } PACK_OFF; typedef enum { - HTTP_METHOD_UNKNOWN = 0, - HTTP_METHOD_OPTIONS, - HTTP_METHOD_GET, - HTTP_METHOD_HEAD, - HTTP_METHOD_POST, - HTTP_METHOD_PUT, - HTTP_METHOD_DELETE, - HTTP_METHOD_TRACE, - HTTP_METHOD_CONNECT + NDPI_HTTP_METHOD_UNKNOWN = 0, + NDPI_HTTP_METHOD_OPTIONS, + NDPI_HTTP_METHOD_GET, + NDPI_HTTP_METHOD_HEAD, + NDPI_HTTP_METHOD_POST, + NDPI_HTTP_METHOD_PUT, + NDPI_HTTP_METHOD_DELETE, + NDPI_HTTP_METHOD_TRACE, + NDPI_HTTP_METHOD_CONNECT } ndpi_http_method; struct ndpi_lru_cache { diff --git a/src/lib/protocols/http.c b/src/lib/protocols/http.c index 4ce80f9c9..661e43bf2 100644 --- a/src/lib/protocols/http.c +++ b/src/lib/protocols/http.c @@ -260,25 +260,25 @@ static void check_content_type_and_change_protocol(struct ndpi_detection_module_ } if(flow->packet.http_method.len < 3) - flow->http.method = HTTP_METHOD_UNKNOWN; + flow->http.method = NDPI_HTTP_METHOD_UNKNOWN; else { switch(flow->packet.http_method.ptr[0]) { - case 'O': flow->http.method = HTTP_METHOD_OPTIONS; break; - case 'G': flow->http.method = HTTP_METHOD_GET; break; - case 'H': flow->http.method = HTTP_METHOD_HEAD; break; + case 'O': flow->http.method = NDPI_HTTP_METHOD_OPTIONS; break; + case 'G': flow->http.method = NDPI_HTTP_METHOD_GET; break; + case 'H': flow->http.method = NDPI_HTTP_METHOD_HEAD; break; case 'P': switch(flow->packet.http_method.ptr[1]) { - case 'O': flow->http.method = HTTP_METHOD_POST; break; - case 'U': flow->http.method = HTTP_METHOD_PUT; break; + case 'O': flow->http.method = NDPI_HTTP_METHOD_POST; break; + case 'U': flow->http.method = NDPI_HTTP_METHOD_PUT; break; } break; - case 'D': flow->http.method = HTTP_METHOD_DELETE; break; - case 'T': flow->http.method = HTTP_METHOD_TRACE; break; - case 'C': flow->http.method = HTTP_METHOD_CONNECT; break; + case 'D': flow->http.method = NDPI_HTTP_METHOD_DELETE; break; + case 'T': flow->http.method = NDPI_HTTP_METHOD_TRACE; break; + case 'C': flow->http.method = NDPI_HTTP_METHOD_CONNECT; break; default: - flow->http.method = HTTP_METHOD_UNKNOWN; + flow->http.method = NDPI_HTTP_METHOD_UNKNOWN; break; } } @@ -949,7 +949,7 @@ void ndpi_search_http_tcp(struct ndpi_detection_module_struct *ndpi_struct, ndpi_http_method ndpi_get_http_method(struct ndpi_detection_module_struct *ndpi_mod, struct ndpi_flow_struct *flow) { if(!flow) - return(HTTP_METHOD_UNKNOWN); + return(NDPI_HTTP_METHOD_UNKNOWN); else return(flow->http.method); } -- cgit v1.2.3 From 1915a63cf29fbe3d2b0a983b2875929518d242ad Mon Sep 17 00:00:00 2001 From: Luca Deri Date: Mon, 1 Apr 2019 12:30:51 +0200 Subject: Implemented ndpi_process_partial_detection() API call to handle partial matches due to the nDPI specified configuration --- example/ndpiReader.c | 31 ++++----- src/include/ndpi_api.h | 15 +++- src/lib/ndpi_main.c | 176 ++++++++++++++++++++++++++++++----------------- src/lib/protocols/dns.c | 6 +- src/lib/protocols/http.c | 4 +- 5 files changed, 149 insertions(+), 83 deletions(-) (limited to 'src/lib/protocols/http.c') diff --git a/example/ndpiReader.c b/example/ndpiReader.c index 4244e3745..72eaecca4 100644 --- a/example/ndpiReader.c +++ b/example/ndpiReader.c @@ -914,11 +914,10 @@ static void node_proto_guess_walker(const void *node, ndpi_VISIT which, int dept struct ndpi_flow_info *flow = *(struct ndpi_flow_info **) node; u_int16_t thread_id = *((u_int16_t *) user_data); - if((which == ndpi_preorder) || (which == ndpi_leaf)) { /* Avoid walking the same node multiple times */ - if((!flow->detection_completed) && flow->ndpi_flow) { + if((which == ndpi_preorder) || (which == ndpi_leaf)) { /* Avoid walking the same node multiple times */ + if((!flow->detection_completed) && flow->ndpi_flow) flow->detected_protocol = ndpi_detection_giveup(ndpi_thread_info[0].workflow->ndpi_struct, flow->ndpi_flow, enable_protocol_guess); - } - + process_ndpi_collected_info(ndpi_thread_info[thread_id].workflow, flow); ndpi_thread_info[thread_id].workflow->stats.protocol_counter[flow->detected_protocol.app_protocol] += flow->src2dst_packets + flow->dst2src_packets; @@ -1956,7 +1955,7 @@ static void printResults(u_int64_t processing_time_usec, u_int64_t setup_time_us printf("\tPeak Memory: %-13s\n", formatBytes(max_ndpi_memory, buf, sizeof(buf))); printf("\tSetup Time: %lu msec\n", (unsigned long)(setup_time_usec/1000)); printf("\tPacket Processing Time: %lu msec\n", (unsigned long)(processing_time_usec/1000)); - + if(!json_flag) { printf("\nTraffic statistics:\n"); printf("\tEthernet bytes: %-13llu (includes ethernet CRC/IFC/trailer)\n", @@ -1992,10 +1991,10 @@ static void printResults(u_int64_t processing_time_usec, u_int64_t setup_time_us float t = (float)(cumulative_stats.ip_packet_count*1000000)/(float)processing_time_usec; float b = (float)(cumulative_stats.total_wire_bytes * 8 *1000000)/(float)processing_time_usec; float traffic_duration; - + if(live_capture) traffic_duration = processing_time_usec; else traffic_duration = (pcap_end.tv_sec*1000000 + pcap_end.tv_usec) - (pcap_start.tv_sec*1000000 + pcap_start.tv_usec); - + printf("\tnDPI throughput: %s pps / %s/sec\n", formatPackets(t, buf), formatTraffic(b, 1, buf1)); t = (float)(cumulative_stats.ip_packet_count*1000000)/(float)traffic_duration; b = (float)(cumulative_stats.total_wire_bytes * 8 *1000000)/(float)traffic_duration; @@ -2005,7 +2004,7 @@ static void printResults(u_int64_t processing_time_usec, u_int64_t setup_time_us strftime(when, sizeof(when), "%d/%b/%Y %H:%M:%S", localtime(&pcap_end.tv_sec)); printf("\tAnalysis end: %s\n", when); printf("\tTraffic throughput: %s pps / %s/sec\n", formatPackets(t, buf), formatTraffic(b, 1, buf1)); - printf("\tTraffic duration: %.3f sec\n", traffic_duration/1000000); + printf("\tTraffic duration: %.3f sec\n", traffic_duration/1000000); } if(enable_protocol_guess) @@ -2490,7 +2489,7 @@ static void ndpi_process_packet(u_char *args, if(memcmp(packet, packet_checked, header->caplen) != 0) printf("INTERNAL ERROR: ingress packet was modified by nDPI: this should not happen [thread_id=%u, packetId=%lu, caplen=%u]\n", thread_id, (unsigned long)ndpi_thread_info[thread_id].workflow->stats.raw_packet_count, header->caplen); - + if((pcap_end.tv_sec-pcap_start.tv_sec) > pcap_analysis_duration) { int i; u_int64_t processing_time_usec, setup_time_usec; @@ -2498,7 +2497,7 @@ static void ndpi_process_packet(u_char *args, gettimeofday(&end, NULL); processing_time_usec = end.tv_sec*1000000 + end.tv_usec - (begin.tv_sec*1000000 + begin.tv_usec); setup_time_usec = begin.tv_sec*1000000 + begin.tv_usec - (startup_time.tv_sec*1000000 + startup_time.tv_usec); - + printResults(processing_time_usec, setup_time_usec); for(i=0; iprefs.num_roots; i++) { @@ -2515,7 +2514,7 @@ static void ndpi_process_packet(u_char *args, memcpy(&pcap_start, &pcap_end, sizeof(pcap_start)); } - /* + /* Leave the free as last statement to avoid crashes when ndpi_detection_giveup() is called above by printResults() */ @@ -2541,7 +2540,7 @@ void * processing_thread(void *_thread_id) { #if defined(linux) && defined(HAVE_PTHREAD_SETAFFINITY_NP) if(core_affinity[thread_id] >= 0) { cpu_set_t cpuset; - + CPU_ZERO(&cpuset); CPU_SET(core_affinity[thread_id], &cpuset); @@ -2559,7 +2558,7 @@ void * processing_thread(void *_thread_id) { struct rte_mbuf *bufs[BURST_SIZE]; u_int16_t num = rte_eth_rx_burst(dpdk_port_id, 0, bufs, BURST_SIZE); u_int i; - + if(num == 0) { usleep(1); continue; @@ -3273,8 +3272,8 @@ int orginal_main(int argc, char **argv) { #else int main(int argc, char **argv) { #endif - int i; - + int i; + if(ndpi_get_api_version() != NDPI_API_VERSION) { printf("nDPI Library version mismatch: please make sure this code and the nDPI library are in sync\n"); return(-1); @@ -3284,7 +3283,7 @@ int orginal_main(int argc, char **argv) { gettimeofday(&startup_time, NULL); ndpi_info_mod = ndpi_init_detection_module(); - + if(ndpi_info_mod == NULL) return -1; memset(ndpi_thread_info, 0, sizeof(ndpi_thread_info)); diff --git a/src/include/ndpi_api.h b/src/include/ndpi_api.h index 386d306fd..84633c3ae 100644 --- a/src/include/ndpi_api.h +++ b/src/include/ndpi_api.h @@ -1,7 +1,7 @@ /* * ndpi_api.h * - * Copyright (C) 2011-17 - ntop.org + * Copyright (C) 2011-19 - ntop.org * * This file is part of nDPI, an open source deep packet inspection * library based on the OpenDPI and PACE technology by ipoque GmbH @@ -206,7 +206,18 @@ extern "C" { */ void ndpi_set_protocol_detection_bitmask2(struct ndpi_detection_module_struct *ndpi_struct, const NDPI_PROTOCOL_BITMASK * detection_bitmask); - + + /** + * Function to be called to see in case of unknown match to see if there is + * a partial match that has been prevented by the current nDPI preferences configuration + * + * @par ndpi_struct = the detection module + * @par flow = the flow given for the detection module + * @return the detected protocol even if the flow is not completed; + * + */ + ndpi_protocol ndpi_get_partial_detection(struct ndpi_detection_module_struct *ndpi_struct, + struct ndpi_flow_struct *flow); /** * Function to be called before we give up with detection for a given flow. * This function reduces the NDPI_UNKNOWN_PROTOCOL detection diff --git a/src/lib/ndpi_main.c b/src/lib/ndpi_main.c index 59de7a763..acbe5b8ec 100644 --- a/src/lib/ndpi_main.c +++ b/src/lib/ndpi_main.c @@ -669,7 +669,7 @@ void ndpi_set_proto_defaults(struct ndpi_detection_module_struct *ndpi_mod, ndpi_mod->proto_defaults[protoId].protoId = protoId, ndpi_mod->proto_defaults[protoId].protoBreed = breed; ndpi_mod->proto_defaults[protoId].can_have_a_subprotocol = can_have_a_subprotocol; - + memcpy(&ndpi_mod->proto_defaults[protoId].master_tcp_protoId, tcp_master_protoId, 2*sizeof(u_int16_t)); memcpy(&ndpi_mod->proto_defaults[protoId].master_udp_protoId, udp_master_protoId, 2*sizeof(u_int16_t)); @@ -677,7 +677,7 @@ void ndpi_set_proto_defaults(struct ndpi_detection_module_struct *ndpi_mod, if(udpDefPorts[j].port_low != 0) addDefaultPort(ndpi_mod, &udpDefPorts[j], &ndpi_mod->proto_defaults[protoId], 0, &ndpi_mod->udpRoot, __FUNCTION__,__LINE__); - + if(tcpDefPorts[j].port_low != 0) addDefaultPort(ndpi_mod, &tcpDefPorts[j], &ndpi_mod->proto_defaults[protoId], 0, &ndpi_mod->tcpRoot, __FUNCTION__,__LINE__); @@ -790,15 +790,15 @@ static int ndpi_string_to_automa(struct ndpi_detection_module_struct *ndpi_struc if(automa->ac_automa == NULL) return(-2); ac_pattern.astring = value, - ac_pattern.rep.number = protocol_id, + ac_pattern.rep.number = protocol_id, ac_pattern.rep.category = (u_int16_t)category, ac_pattern.rep.breed = (u_int16_t)breed; - + #ifdef MATCH_DEBUG printf("Adding to automa [%s][protocol_id: %u][category: %u][breed: %u]\n", value, protocol_id, category, breed); #endif - + if(value == NULL) ac_pattern.length = 0; else @@ -856,14 +856,14 @@ void ndpi_init_protocol_match(struct ndpi_detection_module_struct *ndpi_mod, u_int16_t no_master[2] = { NDPI_PROTOCOL_NO_MASTER_PROTO, NDPI_PROTOCOL_NO_MASTER_PROTO }; ndpi_port_range ports_a[MAX_DEFAULT_PORTS], ports_b[MAX_DEFAULT_PORTS]; static u_int16_t generic_id = NDPI_LAST_IMPLEMENTED_PROTOCOL; - + if(ndpi_mod->proto_defaults[match->protocol_id].protoName == NULL) { if(match->protocol_id == NDPI_PROTOCOL_GENERIC) ndpi_mod->proto_defaults[match->protocol_id].protoName = ndpi_strdup(NDPI_CONST_GENERIC_PROTOCOL_NAME); else ndpi_mod->proto_defaults[match->protocol_id].protoName = ndpi_strdup(match->proto_name); - - ndpi_mod->proto_defaults[match->protocol_id].protoId = match->protocol_id; + + ndpi_mod->proto_defaults[match->protocol_id].protoId = match->protocol_id; ndpi_mod->proto_defaults[match->protocol_id].protoCategory = match->protocol_category; ndpi_mod->proto_defaults[match->protocol_id].protoBreed = match->protocol_breed; @@ -875,7 +875,7 @@ void ndpi_init_protocol_match(struct ndpi_detection_module_struct *ndpi_mod, ndpi_mod->proto_defaults[match->protocol_id].protoName, ndpi_mod->proto_defaults[match->protocol_id].protoCategory, ndpi_build_default_ports(ports_a, 0, 0, 0, 0, 0) /* TCP */, - ndpi_build_default_ports(ports_b, 0, 0, 0, 0, 0) /* UDP */); + ndpi_build_default_ports(ports_b, 0, 0, 0, 0, 0) /* UDP */); } ndpi_add_host_url_subprotocol(ndpi_mod, @@ -1025,7 +1025,7 @@ static void init_string_based_protocols(struct ndpi_detection_module_struct *ndp for(i=0; host_match[i].string_to_match != NULL; i++) ndpi_init_protocol_match(ndpi_mod, &host_match[i]); -#ifdef MATCH_DEBUG +#ifdef MATCH_DEBUG // ac_automata_display(ndpi_mod->host_automa.ac_automa, 'n'); #endif @@ -1065,7 +1065,7 @@ int ndpi_set_detection_preferences(struct ndpi_detection_module_struct *ndpi_mod case ndpi_pref_enable_category_substring_match: ndpi_mod->enable_category_substring_match = (u_int8_t)value; break; - + default: return(-1); } @@ -2089,7 +2089,7 @@ static int ac_match_handler(AC_MATCH_t *m, AC_TEXT_t *txt, AC_REP_t *match) { char buf[64] = { '\0' }; int min_buf_len = (txt->length > 63 /* sizeof(buf)-1 */) ? 63 : txt->length; u_int buf_len = strlen(buf); - + strncpy(buf, txt->astring, min_buf_len); buf[min_buf_len] = '\0'; @@ -2103,8 +2103,8 @@ static int ac_match_handler(AC_MATCH_t *m, AC_TEXT_t *txt, AC_REP_t *match) { char *whatfound = strstr(buf, m->patterns->astring); #ifdef MATCH_DEBUG - printf("[NDPI] %s() [searching=%s][pattern=%s][%s][%c]\n", - __FUNCTION__, buf, m->patterns->astring, + printf("[NDPI] %s() [searching=%s][pattern=%s][%s][%c]\n", + __FUNCTION__, buf, m->patterns->astring, whatfound ? whatfound : "", whatfound[-1]); #endif @@ -2117,7 +2117,7 @@ static int ac_match_handler(AC_MATCH_t *m, AC_TEXT_t *txt, AC_REP_t *match) { if(whatfound && (whatfound != buf) && (m->patterns->astring[0] != '.') /* The searched patter does not start with . */ && strchr(m->patterns->astring, '.') /* The matched pattern has a . (e.g. numeric or sym IPs) */ - && (whatfound[-1] != '.') + && (whatfound[-1] != '.') ) return(0); } @@ -2368,7 +2368,7 @@ struct ndpi_detection_module_struct *ndpi_init_detection_module(void) { #endif ndpi_str->custom_categories.hostnames_hash = NULL; - + ndpi_str->custom_categories.ipAddresses = ndpi_New_Patricia(32 /* IPv4 */); ndpi_str->custom_categories.ipAddresses_shadow = ndpi_New_Patricia(32 /* IPv4 */); @@ -2436,7 +2436,7 @@ int ndpi_match_string(void *_automa, char *string_to_match) { int ndpi_match_string_id(void *_automa, char *string_to_match, unsigned long *id) { AC_TEXT_t ac_input_text; AC_AUTOMATA_t *automa = (AC_AUTOMATA_t*)_automa; - AC_REP_t match = { NDPI_PROTOCOL_UNKNOWN, NDPI_PROTOCOL_CATEGORY_UNSPECIFIED, NDPI_PROTOCOL_UNRATED }; + AC_REP_t match = { NDPI_PROTOCOL_UNKNOWN, NDPI_PROTOCOL_CATEGORY_UNSPECIFIED, NDPI_PROTOCOL_UNRATED }; *id = -1; if((automa == NULL) @@ -2449,7 +2449,7 @@ int ndpi_match_string_id(void *_automa, char *string_to_match, unsigned long *id ac_automata_reset(automa); *id = match.number; - + return(*id != NDPI_PROTOCOL_UNKNOWN ? 0 : -1); } @@ -2476,7 +2476,7 @@ static int hyperscanCustomEventHandler(unsigned int id, int ndpi_match_custom_category(struct ndpi_detection_module_struct *ndpi_struct, char *name, unsigned long *id) { #ifdef DEBUG - printf("[NDPI] %s(%s) [enable_category_substring_match: %u]\n", + printf("[NDPI] %s(%s) [enable_category_substring_match: %u]\n", __FUNCTION__, name, ndpi_struct->enable_category_substring_match); #endif @@ -2575,7 +2575,7 @@ void ndpi_exit_detection_module(struct ndpi_detection_module_struct *ndpi_struct if(ndpi_struct->protocols_ptree) ndpi_Destroy_Patricia((patricia_tree_t*)ndpi_struct->protocols_ptree, free_ptree_data); - + if(ndpi_struct->udpRoot != NULL) ndpi_tdestroy(ndpi_struct->udpRoot, ndpi_free); if(ndpi_struct->tcpRoot != NULL) @@ -2621,7 +2621,7 @@ void ndpi_exit_detection_module(struct ndpi_detection_module_struct *ndpi_struct if(ndpi_struct->custom_categories.hostnames_hash) ht_free((hashtable_t*)ndpi_struct->custom_categories.hostnames_hash); - + ndpi_free(ndpi_struct); } } @@ -2634,7 +2634,7 @@ int ndpi_get_protocol_id_master_proto(struct ndpi_detection_module_struct *ndpi_ u_int16_t** udp_master_proto) { if(protocol_id >= (NDPI_MAX_SUPPORTED_PROTOCOLS+NDPI_MAX_NUM_CUSTOM_PROTOCOLS)) { *tcp_master_proto = ndpi_struct->proto_defaults[NDPI_PROTOCOL_UNKNOWN].master_tcp_protoId, - *udp_master_proto = ndpi_struct->proto_defaults[NDPI_PROTOCOL_UNKNOWN].master_udp_protoId; + *udp_master_proto = ndpi_struct->proto_defaults[NDPI_PROTOCOL_UNKNOWN].master_udp_protoId; return(-1); } @@ -2782,7 +2782,7 @@ char * strsep(char **sp, char *sep) /* ******************************************************************** */ -int ndpi_handle_rule(struct ndpi_detection_module_struct *ndpi_mod, +int ndpi_handle_rule(struct ndpi_detection_module_struct *ndpi_mod, char* rule, u_int8_t do_add) { char *at, *proto, *elem; ndpi_proto_defaults_t *def; @@ -2917,7 +2917,7 @@ int ndpi_load_protocols_file(struct ndpi_detection_module_struct *ndpi_mod, char if(buffer == NULL) { NDPI_LOG_ERR(ndpi_mod, "Memory allocation failure"); - goto close_fd; + goto close_fd; } while(fd) { @@ -3820,18 +3820,18 @@ static int ndpi_init_packet_header(struct ndpi_detection_module_struct *ndpi_str && flow->detected_protocol_stack[0] == NDPI_PROTOCOL_UNKNOWN) { u_int8_t backup; u_int16_t backup1, backup2; - + if(flow->http.url) ndpi_free(flow->http.url); if(flow->http.content_type) ndpi_free(flow->http.content_type); backup = flow->num_processed_pkts; backup1 = flow->guessed_protocol_id; - backup2 = flow->guessed_host_protocol_id; + backup2 = flow->guessed_host_protocol_id; memset(flow, 0, sizeof(*(flow))); flow->num_processed_pkts = backup; flow->guessed_protocol_id = backup1; flow->guessed_host_protocol_id = backup2; - + NDPI_LOG_DBG(ndpi_struct, "tcp syn packet for unknown protocol, reset detection state\n"); } @@ -4165,6 +4165,52 @@ static u_int16_t ndpi_guess_host_protocol_id(struct ndpi_detection_module_struct /* ********************************************************************************* */ +static ndpi_protocol ndpi_process_partial_detection(struct ndpi_detection_module_struct *ndpi_struct, + struct ndpi_flow_struct *flow) { + ndpi_protocol ret; + ndpi_protocol_match_result ret_match; + + ret.master_protocol = flow->guessed_protocol_id; + ret.app_protocol = ndpi_match_host_subprotocol(ndpi_struct, flow, + (char *)flow->host_server_name, + strlen((const char*)flow->host_server_name), + &ret_match, + flow->guessed_protocol_id); + ret.category = ret_match.protocol_category; + + if(ret.app_protocol == NDPI_PROTOCOL_UNKNOWN) + ret.app_protocol = ret.master_protocol; + + ndpi_int_change_protocol(ndpi_struct, flow, ret.app_protocol, ret.master_protocol); + return(ret); +} + +/* ********************************************************************************* */ + +/* + You can call this function at any time in case of unknown match to see if there is + a partial match that has been prevented by the current nDPI preferences configuration + */ +ndpi_protocol ndpi_get_partial_detection(struct ndpi_detection_module_struct *ndpi_struct, + struct ndpi_flow_struct *flow) { + if((flow->guessed_protocol_id == NDPI_PROTOCOL_HTTP) + && (ndpi_struct->http_dont_dissect_response == 0) + && (flow->host_server_name[0] != '\0') + && (!NDPI_ISSET(&flow->excluded_protocol_bitmask, flow->guessed_host_protocol_id))) + return(ndpi_process_partial_detection(ndpi_struct, flow)); + else if((flow->guessed_protocol_id == NDPI_PROTOCOL_DNS) + && (ndpi_struct->dns_dont_dissect_response == 0) + && (flow->host_server_name[0] != '\0') + && (!NDPI_ISSET(&flow->excluded_protocol_bitmask, flow->guessed_host_protocol_id))) + return(ndpi_process_partial_detection(ndpi_struct, flow)); + else { + ndpi_protocol ret = { NDPI_PROTOCOL_UNKNOWN, NDPI_PROTOCOL_UNKNOWN, NDPI_PROTOCOL_CATEGORY_UNSPECIFIED }; + return(ret); + } +} + +/* ********************************************************************************* */ + ndpi_protocol ndpi_detection_giveup(struct ndpi_detection_module_struct *ndpi_struct, struct ndpi_flow_struct *flow, u_int8_t enable_guess) { ndpi_protocol ret = { NDPI_PROTOCOL_UNKNOWN, NDPI_PROTOCOL_UNKNOWN, NDPI_PROTOCOL_CATEGORY_UNSPECIFIED }; @@ -4177,12 +4223,19 @@ ndpi_protocol ndpi_detection_giveup(struct ndpi_detection_module_struct *ndpi_st if(flow->guessed_protocol_id == NDPI_PROTOCOL_STUN) goto check_stun_export; - else if((flow->l4.tcp.ssl_seen_client_cert == 1) && (flow->protos.stun_ssl.ssl.client_certificate[0] != '\0')) { + else if((flow->l4.tcp.ssl_seen_client_cert == 1) + && (flow->protos.stun_ssl.ssl.client_certificate[0] != '\0')) { ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_SSL, NDPI_PROTOCOL_UNKNOWN); } else { - if(!enable_guess) - return(ret); - + ndpi_protocol ret_g = ndpi_get_partial_detection(ndpi_struct, flow); + + if(ret_g.master_protocol != NDPI_PROTOCOL_UNKNOWN) + return(ret_g); + else { + if(!enable_guess) + return(ret); + } + if((flow->guessed_protocol_id == NDPI_PROTOCOL_UNKNOWN) && (flow->packet.l4_protocol == IPPROTO_TCP) && (flow->l4.tcp.ssl_stage > 1)) @@ -4196,7 +4249,7 @@ ndpi_protocol ndpi_detection_giveup(struct ndpi_detection_module_struct *ndpi_st && is_udp_guessable_protocol(guessed_host_protocol_id) )) flow->guessed_host_protocol_id = guessed_host_protocol_id = NDPI_PROTOCOL_UNKNOWN; - + /* Ignore guessed protocol if they have been discarded */ if((guessed_protocol_id != NDPI_PROTOCOL_UNKNOWN) // && (guessed_host_protocol_id == NDPI_PROTOCOL_UNKNOWN) @@ -4207,7 +4260,6 @@ ndpi_protocol ndpi_detection_giveup(struct ndpi_detection_module_struct *ndpi_st if((guessed_protocol_id != NDPI_PROTOCOL_UNKNOWN) || (guessed_host_protocol_id != NDPI_PROTOCOL_UNKNOWN)) { - if((guessed_protocol_id == 0) && (flow->protos.stun_ssl.stun.num_binding_requests > 0) && (flow->protos.stun_ssl.stun.num_processed_pkts > 0)) @@ -4246,7 +4298,7 @@ ndpi_protocol ndpi_detection_giveup(struct ndpi_detection_module_struct *ndpi_st else if(ret.app_protocol == NDPI_PROTOCOL_GOOGLE) ret.app_protocol = NDPI_PROTOCOL_HANGOUT; } - + if(enable_guess && (ret.app_protocol == NDPI_PROTOCOL_UNKNOWN) && flow->packet.iph /* Guess only IPv4 */ @@ -4260,7 +4312,7 @@ ndpi_protocol ndpi_detection_giveup(struct ndpi_detection_module_struct *ndpi_st ntohl(flow->packet.iph->daddr), ntohs(flow->packet.udp ? flow->packet.udp->dest : flow->packet.tcp->dest) ); - + ndpi_fill_protocol_category(ndpi_struct, flow, &ret); return(ret); @@ -4353,15 +4405,15 @@ int ndpi_load_hostname_category(struct ndpi_detection_module_struct *ndpi_struct if(ndpi_struct->custom_categories.hostnames_hash) ht_set((hashtable_t*)ndpi_struct->custom_categories.hostnames_hash, name, (u_int16_t)category); - - return(0); + + return(0); } else { AC_PATTERN_t ac_pattern; - + /* printf("===> Loading %s as %u\n", name, category); */ memset(&ac_pattern, 0, sizeof(ac_pattern)); - + #ifdef HAVE_HYPERSCAN { struct hs_list *h = (struct hs_list*)malloc(sizeof(struct hs_list)); @@ -4401,7 +4453,7 @@ int ndpi_load_hostname_category(struct ndpi_detection_module_struct *ndpi_struct ac_automata_add(ndpi_struct->custom_categories.hostnames_shadow.ac_automa, &ac_pattern); #endif } - + return(0); } @@ -4481,16 +4533,16 @@ int ndpi_enable_loaded_categories(struct ndpi_detection_module_struct *ndpi_str) ndpi_str->custom_categories.hostnames_shadow.ac_automa = ac_automata_init(ac_match_handler); #endif } - + if(ndpi_str->custom_categories.ipAddresses != NULL) - ndpi_Destroy_Patricia((patricia_tree_t*)ndpi_str->custom_categories.ipAddresses, + ndpi_Destroy_Patricia((patricia_tree_t*)ndpi_str->custom_categories.ipAddresses, free_ptree_data); ndpi_str->custom_categories.ipAddresses = ndpi_str->custom_categories.ipAddresses_shadow; ndpi_str->custom_categories.ipAddresses_shadow = ndpi_New_Patricia(32 /* IPv4 */); ndpi_str->custom_categories.categories_loaded = 1; - + return(0); } @@ -4582,7 +4634,7 @@ ndpi_protocol ndpi_detection_process_packet(struct ndpi_detection_module_struct return(ret); flow->num_processed_pkts++; - + if(flow->server_id == NULL) flow->server_id = dst; /* Default */ if(flow->detected_protocol_stack[0] != NDPI_PROTOCOL_UNKNOWN) goto ret_protocols; @@ -4705,7 +4757,7 @@ ndpi_protocol ndpi_detection_process_packet(struct ndpi_detection_module_struct break; } } - + ndpi_check_flow_func(ndpi_struct, flow, &ndpi_selection_packet); ndpi_fill_protocol_category(ndpi_struct, flow, &ret); @@ -4969,22 +5021,22 @@ void ndpi_parse_packet_line_info(struct ndpi_detection_module_struct *ndpi_struc packet->http_response.ptr = &packet->line[0].ptr[NDPI_STATICSTRING_LEN("HTTP/1.1 ")]; packet->http_response.len = packet->line[0].len - NDPI_STATICSTRING_LEN("HTTP/1.1 "); packet->http_num_headers++; - + /* Set server HTTP response code */ if(packet->payload_packet_len >= 12) { char buf[4]; - + /* Set server HTTP response code */ strncpy(buf, (char*)&packet->payload[9], 3); buf[3] = '\0'; - + flow->http.response_status_code = atoi(buf); /* https://en.wikipedia.org/wiki/List_of_HTTP_status_codes */ if((flow->http.response_status_code < 100) || (flow->http.response_status_code > 509)) flow->http.response_status_code = 0; /* Out of range */ } } - + /* "Server:" header line in HTTP response */ if(packet->line[packet->parsed_lines].len > NDPI_STATICSTRING_LEN("Server:") + 1 && strncasecmp((const char *)packet->line[packet->parsed_lines].ptr, "Server:", NDPI_STATICSTRING_LEN("Server:")) == 0) { @@ -5403,7 +5455,7 @@ void ndpi_int_change_protocol(struct ndpi_detection_module_struct *ndpi_struct, } } } - + ndpi_int_change_flow_protocol(ndpi_struct, flow, upper_detected_protocol, lower_detected_protocol); ndpi_int_change_packet_protocol(ndpi_struct, flow, @@ -5435,7 +5487,7 @@ void ndpi_int_reset_protocol(struct ndpi_flow_struct *flow) { int a; for(a = 0; a < NDPI_PROTOCOL_SIZE; a++) - flow->detected_protocol_stack[a] = NDPI_PROTOCOL_UNKNOWN; + flow->detected_protocol_stack[a] = NDPI_PROTOCOL_UNKNOWN; } } @@ -5657,10 +5709,10 @@ ndpi_protocol ndpi_guess_undetected_protocol(struct ndpi_detection_module_struct ret.app_protocol = rc, ret.master_protocol = ndpi_guess_protocol_id(ndpi_struct, NULL, proto, sport, dport, &user_defined_proto); - + if(ret.app_protocol == ret.master_protocol) ret.master_protocol = NDPI_PROTOCOL_UNKNOWN; - + ret.category = ndpi_get_proto_category(ndpi_struct, ret); return(ret); } @@ -5674,7 +5726,7 @@ ndpi_protocol ndpi_guess_undetected_protocol(struct ndpi_detection_module_struct ; else { ret.app_protocol = rc; - + if(rc == NDPI_PROTOCOL_SSL) goto check_guessed_skype; else { @@ -5697,7 +5749,7 @@ ndpi_protocol ndpi_guess_undetected_protocol(struct ndpi_detection_module_struct ret.app_protocol = ndpi_guess_protocol_id(ndpi_struct, NULL, proto, sport, dport, &user_defined_proto); - ret.category = ndpi_get_proto_category(ndpi_struct, ret); + ret.category = ndpi_get_proto_category(ndpi_struct, ret); return(ret); } @@ -5939,7 +5991,7 @@ char* ndpi_strnstr(const char *s, const char *find, size_t slen) { } while(strncmp(s, find, len) != 0); s--; } - + return((char *)s); } @@ -5987,7 +6039,7 @@ int ndpi_match_string_subprotocol(struct ndpi_detection_module_struct *ndpi_stru ndpi_automa *automa = is_host_match ? &ndpi_struct->host_automa : &ndpi_struct->content_automa; AC_REP_t match = { NDPI_PROTOCOL_UNKNOWN, NDPI_PROTOCOL_CATEGORY_UNSPECIFIED, NDPI_PROTOCOL_UNRATED }; - + if((automa->ac_automa == NULL) || (string_to_match_len == 0)) return(NDPI_PROTOCOL_UNKNOWN); @@ -5995,15 +6047,15 @@ int ndpi_match_string_subprotocol(struct ndpi_detection_module_struct *ndpi_stru ac_automata_finalize((AC_AUTOMATA_t*)automa->ac_automa); automa->ac_automa_finalized = 1; } - + ac_input_text.astring = string_to_match, ac_input_text.length = string_to_match_len; ac_automata_search(((AC_AUTOMATA_t*)automa->ac_automa), &ac_input_text, &match); ac_automata_reset(((AC_AUTOMATA_t*)automa->ac_automa)); - + ret_match->protocol_id = match.number, ret_match->protocol_category = match.category, ret_match->protocol_breed = match.breed; - + return(match.number); } @@ -6252,9 +6304,9 @@ struct ndpi_lru_cache* ndpi_lru_cache_init(u_int32_t num_entries) { if(!c->entries) { free(c); return(NULL); - } else + } else c->num_entries = num_entries; - + return(c); } diff --git a/src/lib/protocols/dns.c b/src/lib/protocols/dns.c index b99e5a5da..1c2593feb 100644 --- a/src/lib/protocols/dns.c +++ b/src/lib/protocols/dns.c @@ -153,8 +153,10 @@ void ndpi_search_dns(struct ndpi_detection_module_struct *ndpi_struct, struct nd || ((dns_header.authority_rrs > 0) && (dns_header.authority_rrs <= NDPI_MAX_DNS_REQUESTS)) || ((dns_header.additional_rrs > 0) && (dns_header.additional_rrs <= NDPI_MAX_DNS_REQUESTS))) ) { - /* This is a good reply */ - if(ndpi_struct->dns_dont_dissect_response == 0) { + /* This is a good reply: we dissect it both for request and response */ + + /* Leave the statement below commented necessary in case of call to ndpi_get_partial_detection() */ + /* if(ndpi_struct->dns_dont_dissect_response == 0) */ { x++; if(flow->packet.payload[x] != '\0') { diff --git a/src/lib/protocols/http.c b/src/lib/protocols/http.c index fc392c2b7..33ef9e2ed 100644 --- a/src/lib/protocols/http.c +++ b/src/lib/protocols/http.c @@ -157,7 +157,9 @@ static void check_content_type_and_change_protocol(struct ndpi_detection_module_ } #endif - if(!ndpi_struct->http_dont_dissect_response) { + /* Leave the statement below commented necessary in case of call to ndpi_get_partial_detection() */ + + /* if(!ndpi_struct->http_dont_dissect_response) */ { if((flow->http.url == NULL) && (packet->http_url_name.len > 0) && (packet->host_line.len > 0)) { -- cgit v1.2.3