From fea09e825b8c3872ea54c8361242a9eff538222c Mon Sep 17 00:00:00 2001 From: Luca Deri Date: Fri, 14 Jul 2023 19:52:34 +0200 Subject: Fixes risk mask exception handling while improving the overall performance --- src/include/ndpi_api.h | 2 + src/include/ndpi_typedefs.h | 2 +- src/lib/ndpi_main.c | 7 +++- src/lib/ndpi_utils.c | 94 ++++++++++++++++++++++----------------------- src/lib/protocols/http.c | 54 +++++++++++++++----------- 5 files changed, 87 insertions(+), 72 deletions(-) (limited to 'src') diff --git a/src/include/ndpi_api.h b/src/include/ndpi_api.h index ce124bbf6..e48fc013e 100644 --- a/src/include/ndpi_api.h +++ b/src/include/ndpi_api.h @@ -1082,6 +1082,8 @@ extern "C" { void ndpi_set_tls_cert_expire_days(struct ndpi_detection_module_struct *ndpi_str, u_int8_t days); + void ndpi_handle_risk_exceptions(struct ndpi_detection_module_struct *ndpi_str, + struct ndpi_flow_struct *flow); /* Utility functions to set ndpi malloc/free/print wrappers */ void set_ndpi_malloc(void* (*__ndpi_malloc)(size_t size)); diff --git a/src/include/ndpi_typedefs.h b/src/include/ndpi_typedefs.h index 373a704da..91ae93b4e 100644 --- a/src/include/ndpi_typedefs.h +++ b/src/include/ndpi_typedefs.h @@ -1432,7 +1432,7 @@ struct ndpi_flow_struct { u_int8_t initial_binary_bytes[8], initial_binary_bytes_len; u_int8_t risk_checked:1, ip_risk_mask_evaluated:1, host_risk_mask_evaluated:1, tree_risk_checked:1, _notused:4; ndpi_risk risk_mask; /* Stores the flow risk mask for flow peers */ - ndpi_risk risk; /* Issues found with this flow [bitmask of ndpi_risk] */ + ndpi_risk risk, risk_shadow; /* Issues found with this flow [bitmask of ndpi_risk] */ struct ndpi_risk_information risk_infos[MAX_NUM_RISK_INFOS]; /* String that contains information about the risks found */ u_int8_t num_risk_infos; diff --git a/src/lib/ndpi_main.c b/src/lib/ndpi_main.c index 65c50b6ec..ca7bdd912 100644 --- a/src/lib/ndpi_main.c +++ b/src/lib/ndpi_main.c @@ -6079,7 +6079,6 @@ static u_int32_t make_msteams_key(struct ndpi_flow_struct *flow, u_int8_t use_cl static void ndpi_reconcile_msteams_udp(struct ndpi_detection_module_struct *ndpi_str, struct ndpi_flow_struct *flow, u_int16_t master) { - /* This function can NOT access &ndpi_str->packet since it is called also from ndpi_detection_giveup(), via ndpi_reconcile_protocols() */ if(flow->l4_proto == IPPROTO_UDP) { @@ -6159,6 +6158,12 @@ static void ndpi_reconcile_protocols(struct ndpi_detection_module_struct *ndpi_s // printf("====>> %u.%u [%u]\n", ret->master_protocol, ret->app_protocol, flow->detected_protocol_stack[0]); + if((flow->risk != 0) && (flow->risk != flow->risk_shadow)) { + /* Trick to avoid evaluating exceptions when nothing changed */ + ndpi_handle_risk_exceptions(ndpi_str, flow); + flow->risk_shadow = flow->risk; + } + switch(ret->app_protocol) { case NDPI_PROTOCOL_MICROSOFT_AZURE: ndpi_reconcile_msteams_udp(ndpi_str, flow, flow->detected_protocol_stack[1]); diff --git a/src/lib/ndpi_utils.c b/src/lib/ndpi_utils.c index 75201e55e..5f334081b 100644 --- a/src/lib/ndpi_utils.c +++ b/src/lib/ndpi_utils.c @@ -821,7 +821,7 @@ static const char* ndpi_get_flow_info_by_proto_id(struct ndpi_flow_struct const return flow->host_server_name; break; } - + return NULL; } @@ -831,8 +831,8 @@ const char* ndpi_get_flow_info(struct ndpi_flow_struct const * const flow, ndpi_protocol const * const l7_protocol) { char const * const app_protocol_info = ndpi_get_flow_info_by_proto_id(flow, l7_protocol->app_protocol); - if(app_protocol_info != NULL) - return app_protocol_info; + if(app_protocol_info != NULL) + return app_protocol_info; return ndpi_get_flow_info_by_proto_id(flow, l7_protocol->master_protocol); } @@ -2005,7 +2005,7 @@ const char* ndpi_risk2str(ndpi_risk_enum risk) { case NDPI_DNS_LARGE_PACKET: return("Large DNS Packet (512+ bytes)"); - + case NDPI_DNS_FRAGMENTED: return("Fragmented DNS Message"); @@ -2041,7 +2041,7 @@ const char* ndpi_risk2str(ndpi_risk_enum risk) { case NDPI_MINOR_ISSUES: return("Minor Issues"); - + case NDPI_TCP_ISSUES: return("TCP Connection Issues"); @@ -2069,7 +2069,7 @@ const char* ndpi_severity2str(ndpi_risk_severity s) { case NDPI_RISK_CRITICAL: return("Critical"); - + case NDPI_RISK_EMERGENCY: return("Emergency"); } @@ -2333,7 +2333,7 @@ u_int8_t ndpi_check_issuerdn_risk_exception(struct ndpi_detection_module_struct char *issuerDN) { if(issuerDN != NULL) { ndpi_list *head = ndpi_str->trusted_issuer_dn; - + while(head != NULL) { if(strcmp(issuerDN, head->value) == 0) return(1); /* This is a trusted DN */ @@ -2341,7 +2341,7 @@ u_int8_t ndpi_check_issuerdn_risk_exception(struct ndpi_detection_module_struct head = head->next; } } - + return(0 /* no exception */); } @@ -2356,7 +2356,7 @@ static u_int8_t ndpi_check_hostname_risk_exception(struct ndpi_detection_module_ else { ndpi_automa *automa = &ndpi_str->host_risk_mask_automa; u_int8_t ret = 0; - + if(automa && automa->ac_automa) { AC_TEXT_t ac_input_text; AC_REP_t match; @@ -2364,13 +2364,13 @@ static u_int8_t ndpi_check_hostname_risk_exception(struct ndpi_detection_module_ memset(&match, 0, sizeof(match)); ac_input_text.astring = hostname, ac_input_text.length = strlen(hostname); ac_input_text.option = 0; - + if(ac_automata_search(automa->ac_automa, &ac_input_text, &match) > 0) { if(flow) flow->risk_mask &= match.number64; ret = 1; } } - + return(ret); } } @@ -2383,37 +2383,34 @@ static u_int8_t ndpi_check_ipv4_exception(struct ndpi_detection_module_struct *n u_int32_t addr) { struct in_addr pin; u_int64_t r; - + pin.s_addr = addr; r = ndpi_host_ip_risk_ptree_match(ndpi_str, &pin); - + if(flow) flow->risk_mask &= r; - + return((r != (u_int64_t)-1) ? 1 : 0); } /* ********************************************************************************* */ -static void ndpi_handle_risk_exceptions(struct ndpi_detection_module_struct *ndpi_str, - struct ndpi_flow_struct *flow) { - char *host; - +void ndpi_handle_risk_exceptions(struct ndpi_detection_module_struct *ndpi_str, + struct ndpi_flow_struct *flow) { if(flow->risk == 0) return; /* Nothing to do */ - host = ndpi_get_flow_name(flow); - - if((!flow->host_risk_mask_evaluated) && (!flow->ip_risk_mask_evaluated)) { + if((!flow->host_risk_mask_evaluated) && (!flow->ip_risk_mask_evaluated)) flow->risk_mask = (u_int64_t)-1; /* No mask */ - } if(!flow->host_risk_mask_evaluated) { + char *host = ndpi_get_flow_name(flow); + if(host && (host[0] != '\0')) { /* Check host exception */ ndpi_check_hostname_risk_exception(ndpi_str, flow, host); if(flow->risk_mask == 0) { u_int i; - + /* Might be that the exception applied when some risks were already triggered: we need to clean them up @@ -2424,10 +2421,10 @@ static void ndpi_handle_risk_exceptions(struct ndpi_detection_module_struct *ndp flow->risk_infos[i].info = NULL; } } - + flow->num_risk_infos = 0; } - + /* Used to avoid double checks (e.g. in DNS req/rsp) */ flow->host_risk_mask_evaluated = 1; } @@ -2451,14 +2448,16 @@ static void ndpi_handle_risk_exceptions(struct ndpi_detection_module_struct *ndp void ndpi_set_risk(struct ndpi_detection_module_struct *ndpi_str, struct ndpi_flow_struct *flow, ndpi_risk_enum r, char *risk_message) { + /* Check if the risk is not yet set */ if(!ndpi_isset_risk(ndpi_str, flow, r)) { ndpi_risk v = 1ull << r; - + // NDPI_SET_BIT(flow->risk, (u_int32_t)r); flow->risk |= v; - - ndpi_handle_risk_exceptions(ndpi_str, flow); + + /* Will be handled by ndpi_reconcile_protocols() */ + // ndpi_handle_risk_exceptions(ndpi_str, flow); if(flow->risk != 0 /* check if it has been masked */) { if(risk_message != NULL) { @@ -2574,15 +2573,15 @@ int ndpi_is_valid_hostname(char * const str, size_t len) { || (str[i] == '_') || (str[i] == ':') ) - continue; /* Used in hostnames */ - else if((ndpi_isprint(str[i]) == 0) + continue; /* Used in hostnames */ + else if((ndpi_isprint(str[i]) == 0) || ndpi_isspace(str[i]) || ndpi_ispunct(str[i]) ) { return(0); } } - + return(1); } @@ -2763,7 +2762,7 @@ u_int32_t ndpi_get_flow_error_code(struct ndpi_flow_struct *flow) { case NDPI_PROTOCOL_HTTP: return(flow->http.response_status_code); - + case NDPI_PROTOCOL_SNMP: return(flow->protos.snmp.error_status); } @@ -2823,7 +2822,7 @@ char* ndpi_get_flow_risk_info(struct ndpi_flow_struct *flow, char *out, u_int out_len, u_int8_t use_json) { u_int i, offset = 0; - + if((out == NULL) || (flow == NULL) || (flow->num_risk_infos == 0)) @@ -2833,15 +2832,15 @@ char* ndpi_get_flow_risk_info(struct ndpi_flow_struct *flow, ndpi_serializer serializer; u_int32_t buffer_len; char *buffer; - + if(ndpi_init_serializer(&serializer, ndpi_serialization_format_json) == -1) return(NULL); for(i=0; inum_risk_infos; i++) ndpi_serialize_uint32_string(&serializer, - flow->risk_infos[i].id, - flow->risk_infos[i].info); - + flow->risk_infos[i].id, + flow->risk_infos[i].info); + buffer = ndpi_serializer_get_buffer(&serializer, &buffer_len); if(buffer && (buffer_len > 0)) { @@ -2850,28 +2849,28 @@ char* ndpi_get_flow_risk_info(struct ndpi_flow_struct *flow, strncpy(out, buffer, l); out[l] = '\0'; } - + ndpi_term_serializer(&serializer); return(out); } else { out[0] = '\0', out_len--; - + for(i=0; (inum_risk_infos) && (out_len > offset); i++) { int rc = snprintf(&out[offset], out_len-offset, "%s%s", (i == 0) ? "" : " / ", flow->risk_infos[i].info); - + if(rc <= 0) break; else offset += rc; } - + if(offset > out_len) offset = out_len; - + out[offset] = '\0'; - + return(out[0] == '\0' ? NULL : out); } } @@ -2879,7 +2878,7 @@ char* ndpi_get_flow_risk_info(struct ndpi_flow_struct *flow, /* ******************************************* */ /* This function checks if a flow having the specified risk - parameters is an exception (i.e. the flow risk should not + parameters is an exception (i.e. the flow risk should not be triggered) or not. You can use this function to check if a flow that @@ -2899,7 +2898,7 @@ u_int8_t ndpi_check_flow_risk_exceptions(struct ndpi_detection_module_struct *nd if(ndpi_check_hostname_risk_exception(ndpi_str, NULL, (char*)params[i].value)) return(1); break; - + case NDPI_PARAM_ISSUER_DN: if(ndpi_check_issuerdn_risk_exception(ndpi_str, (char*)params[i].value)) return(1); @@ -2920,7 +2919,7 @@ u_int8_t ndpi_check_flow_risk_exceptions(struct ndpi_detection_module_struct *nd break; } } - + return(0); } @@ -2994,7 +2993,7 @@ char* ndpi_intoav4(unsigned int addr, char* buf, u_int16_t bufLen) { /* Find the nearest (>=) value of x */ u_int32_t ndpi_nearest_power_of_two(u_int32_t x) { x--; - + x |= x >> 1; x |= x >> 2; x |= x >> 4; @@ -3004,4 +3003,3 @@ u_int32_t ndpi_nearest_power_of_two(u_int32_t x) { x++; return(x); } - diff --git a/src/lib/protocols/http.c b/src/lib/protocols/http.c index 8885906d9..6951dd85c 100644 --- a/src/lib/protocols/http.c +++ b/src/lib/protocols/http.c @@ -741,28 +741,38 @@ static void ndpi_check_http_url(struct ndpi_detection_module_struct *ndpi_struct static void ndpi_check_http_server(struct ndpi_detection_module_struct *ndpi_struct, struct ndpi_flow_struct *flow, const char *server, u_int server_len) { - if(server_len > 7) { - u_int off; - - if((strncasecmp(server, "Apache/", off = 7) == 0) /* X.X.X */ - || (strncasecmp(server, "nginx/", off = 6) == 0) /* X.X.X */) { - u_int i, j, a, b, c; - char buf[16] = { '\0' }; - - for(i=off, j=0; (i 7) { + u_int off, i; + + if((strncasecmp(server, "Apache/", off = 7) == 0) /* X.X.X */ + || (strncasecmp(server, "nginx/", off = 6) == 0) /* X.X.X */) { + u_int j, a, b, c; + char buf[16] = { '\0' }; + + for(i=off, j=0; (i