diff options
Diffstat (limited to 'example/ndpiReader.c')
| -rw-r--r-- | example/ndpiReader.c | 573 |
1 files changed, 387 insertions, 186 deletions
diff --git a/example/ndpiReader.c b/example/ndpiReader.c index 9e4c67b60..d164f5159 100644 --- a/example/ndpiReader.c +++ b/example/ndpiReader.c @@ -1,7 +1,7 @@ /* * ndpiReader.c * - * Copyright (C) 2011-24 - ntop.org + * Copyright (C) 2011-25 - ntop.org * * nDPI is free software: you can redistribute it and/or modify * it under the terms of the GNU Lesser General Public License as published by @@ -68,6 +68,9 @@ #define HEURISTICS_CODE 1 +/* Necessary to make sure protocols are properly defined */ +#define PROTO_DEBUG 1 + /** Client parameters **/ static char *_pcap_file[MAX_NUM_READER_THREADS]; /**< Ingress pcap file/interfaces */ @@ -92,6 +95,7 @@ static ndpi_serialization_format serialization_format = ndpi_serialization_forma static char* domain_to_check = NULL; static char* ip_port_to_check = NULL; static u_int8_t ignore_vlanid = 0; +extern char *protocolsDirPath; /**< Directory containing protocol files */ FILE *fingerprint_fp = NULL; /**< for flow fingerprint export */ #ifdef __linux__ static char *bind_mask = NULL; @@ -128,6 +132,7 @@ u_int8_t max_num_udp_dissected_pkts = 24 /* 8 is enough for most protocols, Sign static u_int32_t pcap_analysis_duration = (u_int32_t)-1; static u_int32_t risk_stats[NDPI_MAX_RISK] = { 0 }, risks_found = 0, flows_with_risks = 0; static struct ndpi_stats cumulative_stats; +static int cumulative_stats_initialized = 0; static u_int16_t decode_tunnels = 0; static u_int16_t num_loops = 1; static u_int8_t shutdown_app = 0, quiet_mode = 0; @@ -158,6 +163,8 @@ int malloc_size_stats = 0; int monitoring_enabled; +char *protocolsDirPath; + struct flow_info { struct ndpi_flow_info *flow; u_int16_t thread_id; @@ -239,7 +246,6 @@ struct receiver *receivers = NULL, *topReceivers = NULL; #define WIRESHARK_METADATA_SERVERNAME 0x01 #define WIRESHARK_METADATA_JA4C 0x02 -#define WIRESHARK_METADATA_TLS_HEURISTICS_MATCHING_FINGERPRINT 0x03 struct ndpi_packet_tlv { u_int16_t type; @@ -299,9 +305,8 @@ static int dpdk_port_id = 0, dpdk_run_capture = 1; void test_lib(); /* Forward */ extern void ndpi_report_payload_stats(FILE *out); -extern int parse_proto_name_list(char *str, NDPI_PROTOCOL_BITMASK *bitmask, - int inverted_logic); extern u_int8_t is_ndpi_proto(struct ndpi_flow_info *flow, u_int16_t id); +static char const *ndpi_cfg_error2string(ndpi_cfg_error const err); /* ********************************** */ @@ -397,6 +402,40 @@ static u_int check_bin_doh_similarity(struct ndpi_bin *bin, float *similarity) { /* *********************************************** */ +static char _proto_delim[] = " \t,:;"; +static int enable_disable_protocols_list(struct ndpi_detection_module_struct *ndpi_str, char *str, int inverted_logic) { + char *n; + char op; + ndpi_cfg_error rc; + + if(!inverted_logic) + op = 1; /* Default action: enable protocol */ + else + op = 0; /* Default action: disable protocol */ + + for(n = strtok(str,_proto_delim); n && *n; n = strtok(NULL,_proto_delim)) { + if(*n == '-') { + op = !inverted_logic ? 0 : 1; + n++; + } else if(*n == '+') { + op = !inverted_logic ? 1 : 0; + n++; + } + if(op) + rc = ndpi_set_config(ndpi_str, n, "enable", "1"); + else + rc = ndpi_set_config(ndpi_str, n, "enable", "0"); + if(rc != NDPI_CFG_OK) { + LOG(NDPI_LOG_ERROR, "Error enabling/disabling protocol [%s]: %s (%d)\n", + n, ndpi_cfg_error2string(rc), rc); + } + } + + return 0; +} + +/* *********************************************** */ + void ndpiCheckHostStringMatch(char *testChar) { ndpi_protocol_match_result match = { NDPI_PROTOCOL_UNKNOWN, NDPI_PROTOCOL_CATEGORY_UNSPECIFIED, NDPI_PROTOCOL_UNRATED }; @@ -404,14 +443,11 @@ void ndpiCheckHostStringMatch(char *testChar) { char appBufStr[64]; ndpi_protocol detected_protocol; struct ndpi_detection_module_struct *ndpi_str; - NDPI_PROTOCOL_BITMASK all; if(!testChar) return; ndpi_str = ndpi_init_detection_module(NULL); - NDPI_BITMASK_SET_ALL(all); - ndpi_set_protocol_detection_bitmask2(ndpi_str, &all); ndpi_finalize_initialization(ndpi_str); testRes = ndpi_match_string_subprotocol(ndpi_str, @@ -473,14 +509,11 @@ static void ndpiCheckIPMatch(char *testChar) { ndpi_protocol detected_protocol; int i; ndpi_cfg_error rc; - NDPI_PROTOCOL_BITMASK all; if(!testChar) return; ndpi_str = ndpi_init_detection_module(NULL); - NDPI_BITMASK_SET_ALL(all); - ndpi_set_protocol_detection_bitmask2(ndpi_str, &all); if(_protoFilePath != NULL) ndpi_load_protocols_file(ndpi_str, _protoFilePath); @@ -645,90 +678,87 @@ static void help(u_int long_help) { " [-r <file>][-R][-j <file>][-S <file>][-T <num>][-U <num>] [-x <domain>]\n" " [-a <mode>][-B proto_list][-L <domain suffixes>]\n\n" "Usage:\n" - " -i <file.pcap|device> | Specify a pcap file/playlist to read packets from or a\n" - " | device for live capture (comma-separated list)\n" - " -f <BPF filter> | Specify a BPF filter for filtering selected traffic\n" - " -s <duration> | Maximum capture duration in seconds (live traffic capture only)\n" - " -m <duration> | Split analysis duration in <duration> max seconds\n" - " -p <file>.protos | Specify a protocol file (eg. protos.txt)\n" - " -l <num loops> | Number of detection loops (test only)\n" - " -L <domain suffixes> | Domain suffixes (e.g. ../lists/public_suffix_list.dat)\n" - " -n <num threads> | Number of threads. Default: number of interfaces in -i.\n" - " | Ignored with pcap files.\n" - " -N <path> | Address cache dump/restore pathxo.\n" - " -b <num bin clusters> | Number of bin clusters\n" - " -k <file> | Specify a file to write serialized detection results\n" - " -K <format> | Specify the serialization format for `-k'\n" - " | Valid formats are tlv, csv or json (default)\n" + " -i <file.pcap|device> | Specify a pcap file/playlist to read packets from or a\n" + " | device for live capture (comma-separated list)\n" + " -f <BPF filter> | Specify a BPF filter for filtering selected traffic\n" + " -s <duration> | Maximum capture duration in seconds (live traffic capture only)\n" + " -m <duration> | Split analysis duration in <duration> max seconds\n" + " -p <file>.protos | Specify a protocol file (eg. protos.txt)\n" + " -l <num loops> | Number of detection loops (test only)\n" + " -L <domain suffixes> | Domain suffixes (e.g. ../lists/public_suffix_list.dat)\n" + " -n <num threads> | Number of threads. Default: number of interfaces in -i.\n" + " | Ignored with pcap files.\n" + " -N <path> | Address cache dump/restore pathxo.\n" + " -b <num bin clusters> | Number of bin clusters\n" + " -k <file> | Specify a file to write serialized detection results\n" + " -K <format> | Specify the serialization format for `-k'\n" + " | Valid formats are tlv, csv or json (default)\n" #ifdef __linux__ - " -g <id:id...> | Thread affinity mask (one core id per thread)\n" + " -g <id:id...> | Thread affinity mask (one core id per thread)\n" #endif - " -a <mode> | Generates option values for GUIs\n" - " | 0 - List known protocols\n" - " | 1 - List known categories\n" - " | 2 - List known risks\n" - " -d | Disable protocol guess (by ip and by port) and use only DPI.\n" - " | It is a shortcut to --cfg=dpi.guess_on_giveup,0\n" - " -e <len> | Min human readeable string match len. Default %u\n" - " -q | Quiet mode\n" - " -F | Enable flow stats\n" - " -t | Dissect GTP/TZSP tunnels\n" - " -P <a>:<b>:<c>:<d>:<e> | Enable payload analysis:\n" - " | <a> = min pattern len to search\n" - " | <b> = max pattern len to search\n" - " | <c> = max num packets per flow\n" - " | <d> = max packet payload dissection\n" - " | <d> = max num reported payloads\n" - " | Default: %u:%u:%u:%u:%u\n" - " -c <path> | Load custom categories from the specified file\n" - " -C <path> | Write output in CSV format on the specified file\n" - " -E <path> | Write flow fingerprints on the specified file\n" - " -r <path> | Load risky domain file\n" - " -R | Print detected realtime protocols\n" - " -j <path> | Load malicious JA4 fingeprints\n" - " -S <path> | Load malicious SSL certificate SHA1 fingerprints\n" - " -G <dir> | Bind domain names to categories loading files from <dir>\n" - " -w <path> | Write test output on the specified file. This is useful for\n" - " | testing purposes in order to compare results across runs\n" - " -h | This help\n" - " -H | This help plus some information about supported protocols/risks\n" - " -v <1|2|3|4> | Verbose 'unknown protocol' packet print.\n" - " | 1 = verbose\n" - " | 2 = very verbose\n" - " | 3 = port stats\n" - " | 4 = hash stats\n" - " -V <0-4> | nDPI logging level\n" - " | 0 - error, 1 - trace, 2 - debug, 3 - extra debug\n" - " | >3 - extra debug + log enabled for all protocols (i.e. '-u all')\n" - " -u all|proto|num[,...] | Enable logging only for such protocol(s)\n" - " | If this flag is present multiple times (directly, or via '-V'),\n" - " | only the last instance will be considered\n" - " -B all|proto|num[,...] | Disable such protocol(s). By defaul all protocols are enabled\n" - " -T <num> | Max number of TCP processed packets before giving up [default: %u]\n" - " -U <num> | Max number of UDP processed packets before giving up [default: %u]\n" - " -D | Enable DoH traffic analysis based on content (no DPI)\n" - " -x <domain> | Check domain name [Test only]\n" - " -I | Ignore VLAN id for flow hash calculation\n" - " -A | Dump internal statistics (LRU caches / Patricia trees / Ahocarasick automas / ...\n" - " -M | Memory allocation stats on data-path (only by the library).\n" - " | It works only on single-thread configuration\n" - " --openvp_heuristics | Enable OpenVPN heuristics.\n" - " | It is a shortcut to --cfg=openvpn,dpi.heuristics,0x01\n" - " --tls_heuristics | Enable TLS heuristics.\n" - " | It is a shortcut to --cfg=tls,dpi.heuristics,0x07\n" - " --cfg=proto,param,value | Configure the specific attribute of this protocol\n" - " --dump-fpc-stats | Print FPC statistics\n" + " -a <mode> | Generates option values for GUIs\n" + " | 0 - List known protocols\n" + " | 1 - List known categories\n" + " | 2 - List known risks\n" + " -d | Disable protocol guess (by ip and by port) and use only DPI.\n" + " | It is a shortcut to --cfg=dpi.guess_on_giveup,0\n" + " -e <len> | Min human readeable string match len. Default %u\n" + " -q | Quiet mode\n" + " -F | Enable flow stats\n" + " -t | Dissect GTP/TZSP tunnels\n" + " -P <a>:<b>:<c>:<d>:<e> | Enable payload analysis:\n" + " | <a> = min pattern len to search\n" + " | <b> = max pattern len to search\n" + " | <c> = max num packets per flow\n" + " | <d> = max packet payload dissection\n" + " | <e> = max num reported payloads\n" + " | Default: %u:%u:%u:%u:%u\n" + " -c <path> | Load custom categories from the specified file\n" + " -C <path> | Write output in CSV format on the specified file\n" + " -E <path> | Write flow fingerprints on the specified file\n" + " -r <path> | Load risky domain file\n" + " -R | Print detected realtime protocols\n" + " -j <path> | Load malicious JA4 fingeprints\n" + " -S <path> | Load malicious SSL certificate SHA1 fingerprints\n" + " -G <dir> | Bind domain names to categories loading files from <dir>\n" + " -w <path> | Write test output on the specified file. This is useful for\n" + " | testing purposes in order to compare results across runs\n" + " --protocols-list-dir <dir> | Directory containing protocols directory (e.g. ../lists/protocols)\n" + " -h | This help\n" + " -H | This help plus some information about supported protocols/risks\n" + " -v <1|2|3|4> | Verbose 'unknown protocol' packet print.\n" + " | 1 = verbose\n" + " | 2 = very verbose\n" + " | 3 = port stats\n" + " | 4 = hash stats\n" + " -V <0-4> | nDPI logging level\n" + " | 0 - error, 1 - trace, 2 - debug, 3 - extra debug\n" + " | >3 - extra debug + log enabled for all protocols (i.e. '-u all')\n" + " -u all|proto|num[,...] | Enable logging only for such protocol(s)\n" + " | If this flag is present multiple times (directly, or via '-V'),\n" + " | only the last instance will be considered\n" + " -B all|proto|num[,...] | Disable such protocol(s). By defaul all protocols are enabled\n" + " -T <num> | Max number of TCP processed packets before giving up [default: %u]\n" + " -U <num> | Max number of UDP processed packets before giving up [default: %u]\n" + " -D | Enable DoH traffic analysis based on content (no DPI)\n" + " -x <domain> | Check domain name [Test only]\n" + " -I | Ignore VLAN id for flow hash calculation\n" + " -A | Dump internal statistics (LRU caches / Patricia trees / Ahocarasick automas / ...\n" + " -M | Memory allocation stats on data-path (only by the library).\n" + " | It works only on single-thread configuration\n" + " --openvp_heuristics | Enable OpenVPN heuristics.\n" + " | It is a shortcut to --cfg=openvpn,dpi.heuristics,0x01\n" + " --tls_heuristics | Enable TLS heuristics.\n" + " | It is a shortcut to --cfg=tls,dpi.heuristics,0x07\n" + " --cfg=proto,param,value | Configure the specific attribute of this protocol\n" + " --dump-fpc-stats | Print FPC statistics\n" , human_readeable_string_len, min_pattern_len, max_pattern_len, max_num_packets_per_flow, max_packet_payload_dissection, max_num_reported_top_payloads, max_num_tcp_dissected_pkts, max_num_udp_dissected_pkts); - NDPI_PROTOCOL_BITMASK all; struct ndpi_detection_module_struct *ndpi_str = ndpi_init_detection_module(NULL); - NDPI_BITMASK_SET_ALL(all); - ndpi_set_protocol_detection_bitmask2(ndpi_str, &all); - if(_protoFilePath != NULL) ndpi_load_protocols_file(ndpi_str, _protoFilePath); @@ -759,8 +789,8 @@ static void help(u_int long_help) { sizeof(((struct ndpi_flow_struct *)0)->protos)); printf("\n\nnDPI supported protocols:\n"); - printf("%3s %8s %-22s %-10s %-8s %-12s %-18s %-31s %-31s \n", - "Id", "Userd-id", "Protocol", "Layer_4", "Nw_Proto", "Breed", "Category","Def UDP Port/s","Def TCP Port/s"); + printf("%3s %8s %-26s %-10s %-8s %-21s %-18s %-31s %-31s %6s\n", + "Id", "Userd-id", "Protocol", "Layer_4", "Nw_Proto", "Breed", "Category","Def UDP Port/s","Def TCP Port/s", "Custom"); num_threads = 1; ndpi_dump_protocols(ndpi_str, stdout); @@ -823,6 +853,7 @@ static struct option longopts[] = { { "payload-analysis", required_argument, NULL, 'P'}, { "result-path", required_argument, NULL, 'w'}, { "quiet", no_argument, NULL, 'q'}, + { "protocols-list-dir", required_argument, NULL, 180}, { "cfg", required_argument, NULL, OPTLONG_VALUE_CFG}, { "openvpn_heuristics", no_argument, NULL, OPTLONG_VALUE_OPENVPN_HEURISTICS}, @@ -900,17 +931,13 @@ void extcap_config() { u_int ndpi_num_supported_protocols; int i; ndpi_proto_defaults_t *proto_defaults; - NDPI_PROTOCOL_BITMASK all; struct ndpi_detection_module_struct *ndpi_str = ndpi_init_detection_module(NULL); if(!ndpi_str) exit(0); - NDPI_BITMASK_SET_ALL(all); - ndpi_set_protocol_detection_bitmask2(ndpi_str, &all); - - ndpi_finalize_initialization(ndpi_str); + if(ndpi_finalize_initialization(ndpi_str) != 0) exit(0); - ndpi_num_supported_protocols = ndpi_get_ndpi_num_supported_protocols(ndpi_str); + ndpi_num_supported_protocols = ndpi_get_num_protocols(ndpi_str); proto_defaults = ndpi_get_proto_defaults(ndpi_str); /* -i <interface> */ @@ -1490,10 +1517,7 @@ static void parse_parameters(int argc, char **argv) case '9': { struct ndpi_detection_module_struct *ndpi_str = ndpi_init_detection_module(NULL); - NDPI_PROTOCOL_BITMASK all; - NDPI_BITMASK_SET_ALL(all); - ndpi_set_protocol_detection_bitmask2(ndpi_str, &all); ndpi_finalize_initialization(ndpi_str); extcap_packet_filter = ndpi_get_proto_by_name(ndpi_str, optarg); @@ -1529,6 +1553,10 @@ static void parse_parameters(int argc, char **argv) } break; + case 180: + protocolsDirPath = optarg; + break; + default: #ifdef DEBUG_TRACE if(trace) fprintf(trace, " #### Unknown option -%c: skipping it #### \n", opt); @@ -1726,8 +1754,10 @@ static void printFlow(u_int32_t id, struct ndpi_flow_info *flow, u_int16_t threa flow->protocol, f/1000.0, l/1000.0, (l-f)/1000.0, - flow->src_name, ntohs(flow->src_port), - flow->dst_name, ntohs(flow->dst_port) + flow->src_name ? flow->src_name : "", + ntohs(flow->src_port), + flow->dst_name ? flow->dst_name : "", + ntohs(flow->dst_port) ); fprintf(csv_fp, "%s|", @@ -1834,10 +1864,12 @@ static void printFlow(u_int32_t id, struct ndpi_flow_info *flow, u_int16_t threa fprintf(out, "%s%s%s:%u %s %s%s%s:%u ", (flow->ip_version == 6) ? "[" : "", - flow->src_name, (flow->ip_version == 6) ? "]" : "", ntohs(flow->src_port), + flow->src_name ? flow->src_name : "", + (flow->ip_version == 6) ? "]" : "", ntohs(flow->src_port), flow->bidirectional ? "<->" : "->", (flow->ip_version == 6) ? "[" : "", - flow->dst_name, (flow->ip_version == 6) ? "]" : "", ntohs(flow->dst_port) + flow->dst_name ? flow->dst_name : "", + (flow->ip_version == 6) ? "]" : "", ntohs(flow->dst_port) ); if(flow->vlan_id > 0) fprintf(out, "[VLAN: %u]", flow->vlan_id); @@ -1856,6 +1888,26 @@ static void printFlow(u_int32_t id, struct ndpi_flow_info *flow, u_int16_t threa if(flow->tunnel_type != ndpi_no_tunnel) fprintf(out, "%s:", ndpi_tunnel2str(flow->tunnel_type)); +#ifdef PROTO_DEBUG + if((flow->detected_protocol.proto.master_protocol != NDPI_PROTOCOL_UNKNOWN) && + (flow->detected_protocol.proto.app_protocol != NDPI_PROTOCOL_UNKNOWN) + && (flow->detected_protocol.proto.app_protocol != + flow->detected_protocol.proto.master_protocol)) { + if(ndpi_is_master_only_protocol(ndpi_thread_info[thread_id].workflow->ndpi_struct, + flow->detected_protocol.proto.app_protocol)) { + printf("[INTERNAL ERROR] %u/%s [%u.%u/%s] unexpected as application protocol\n", + flow->detected_protocol.proto.app_protocol, + ndpi_get_proto_name(ndpi_thread_info[thread_id].workflow->ndpi_struct, + flow->detected_protocol.proto.app_protocol), + flow->detected_protocol.proto.master_protocol, + flow->detected_protocol.proto.app_protocol, + ndpi_protocol2name(ndpi_thread_info[thread_id].workflow->ndpi_struct, + flow->detected_protocol, buf1, sizeof(buf1)) + ); + } + } +#endif + fprintf(out, "%s/%s][IP: %u/%s]", ndpi_protocol2id(flow->detected_protocol, buf, sizeof(buf)), ndpi_protocol2name(ndpi_thread_info[thread_id].workflow->ndpi_struct, @@ -1870,6 +1922,25 @@ static void printFlow(u_int32_t id, struct ndpi_flow_info *flow, u_int16_t threa fprintf(out, "[Stream Content: %s]", ndpi_multimedia_flowtype2str(content, sizeof(content), flow->multimedia_flow_types)); } + if((flow->detected_protocol.proto.master_protocol == NDPI_PROTOCOL_RTP) || (flow->detected_protocol.proto.app_protocol == NDPI_PROTOCOL_RTP)) + { + if (flow->rtp[0 /* cli -> srv */].payload_detected || flow->rtp[1].payload_detected) { + fprintf(out, "[Payload Type: "); + + if (flow->rtp[0].payload_detected) + fprintf(out, "%s (%u.%u)", + ndpi_rtp_payload_type2str(flow->rtp[0].payload_type, flow->rtp[0].evs_subtype), flow->rtp[0].payload_type, flow->rtp[0].evs_subtype); + + if(flow->rtp[1 /* srv -> cli */].payload_detected) { + if (flow->rtp[0].payload_detected) fprintf(out, " / "); + + fprintf(out, "%s (%u.%u)]", + ndpi_rtp_payload_type2str(flow->rtp[1].payload_type, flow->rtp[1].evs_subtype), flow->rtp[1].payload_type, flow->rtp[1].evs_subtype); + } else + fprintf(out, "]"); + } + } + fprintf(out, "[%s]", ndpi_is_encrypted_proto(ndpi_thread_info[thread_id].workflow->ndpi_struct, flow->detected_protocol) ? "Encrypted" : "ClearText"); @@ -2036,6 +2107,22 @@ static void printFlow(u_int32_t id, struct ndpi_flow_info *flow, u_int16_t threa } } break; + + case INFO_FASTCGI: + if (flow->fast_cgi.url[0] != '\0') + { + fprintf(out, "[Url: %s]", flow->fast_cgi.url); + } + if (flow->fast_cgi.user_agent[0] != '\0') + { + fprintf(out, "[User-agent: %s]", flow->fast_cgi.user_agent); + } + break; + + case INFO_BFCP: + fprintf(out, "[Conference Id: %d]", flow->bfcp.conference_id); + fprintf(out, "[User Id: %d]", flow->bfcp.user_id); + break; } if(flow->ssh_tls.advertised_alpns) @@ -2050,6 +2137,8 @@ static void printFlow(u_int32_t id, struct ndpi_flow_info *flow, u_int16_t threa if(flow->mining.currency[0] != '\0') fprintf(out, "[currency: %s]", flow->mining.currency); if(flow->dns.geolocation_iata_code[0] != '\0') fprintf(out, "[GeoLocation: %s]", flow->dns.geolocation_iata_code); + if(flow->dns.transaction_id != 0) fprintf(out, "[DNS Id: 0x%.4x]", flow->dns.transaction_id); + if(flow->dns.ptr_domain_name[0] != '\0') fprintf(out, "[DNS Ptr: %s]", flow->dns.ptr_domain_name); if((flow->src2dst_packets+flow->dst2src_packets) > 5) { if(flow->iat_c_to_s && flow->iat_s_to_c) { @@ -2087,14 +2176,8 @@ static void printFlow(u_int32_t id, struct ndpi_flow_info *flow, u_int16_t threa if(flow->num_packets_before_monitoring > 0) fprintf(out, "[RTP packets: %d/%d]", flow->stun.rtp_counters[0], flow->stun.rtp_counters[1]); - if(flow->http.url[0] != '\0') { - ndpi_risk_enum risk = ndpi_validate_url(flow->http.url); - - if(risk != NDPI_NO_RISK) - NDPI_SET_BIT(flow->risk, risk); - - fprintf(out, "[URL: %s]", flow->http.url); - } + if(flow->http.url[0] != '\0') + fprintf(out, "[URL: %s]", flow->http.url); if(flow->http.response_status_code) fprintf(out, "[StatusCode: %u]", flow->http.response_status_code); @@ -2410,6 +2493,7 @@ static void node_print_known_proto_walker(const void *node, static void node_proto_guess_walker(const void *node, ndpi_VISIT which, int depth, void *user_data) { struct ndpi_flow_info *flow = *(struct ndpi_flow_info **) node; u_int16_t thread_id = *((u_int16_t *) user_data), proto, fpc_proto; + ndpi_protocol_category_t category; (void)depth; @@ -2435,6 +2519,8 @@ static void node_proto_guess_walker(const void *node, ndpi_VISIT which, int dept fpc_proto = flow->fpc.proto.app_protocol ? flow->fpc.proto.app_protocol : flow->fpc.proto.master_protocol; fpc_proto = ndpi_map_user_proto_id_to_ndpi_id(ndpi_thread_info[thread_id].workflow->ndpi_struct, fpc_proto); + category = flow->detected_protocol.category; + ndpi_thread_info[thread_id].workflow->stats.protocol_counter[proto] += flow->src2dst_packets + flow->dst2src_packets; ndpi_thread_info[thread_id].workflow->stats.protocol_counter_bytes[proto] += flow->src2dst_bytes + flow->dst2src_bytes; ndpi_thread_info[thread_id].workflow->stats.protocol_flows[proto]++; @@ -2444,6 +2530,9 @@ static void node_proto_guess_walker(const void *node, ndpi_VISIT which, int dept ndpi_thread_info[thread_id].workflow->stats.fpc_protocol_counter_bytes[fpc_proto] += flow->src2dst_bytes + flow->dst2src_bytes; ndpi_thread_info[thread_id].workflow->stats.fpc_protocol_flows[fpc_proto]++; ndpi_thread_info[thread_id].workflow->stats.fpc_flow_confidence[flow->fpc.confidence]++; + ndpi_thread_info[thread_id].workflow->stats.category_counter[category] += flow->src2dst_packets + flow->dst2src_packets; + ndpi_thread_info[thread_id].workflow->stats.category_counter_bytes[category] += flow->src2dst_bytes + flow->dst2src_bytes; + ndpi_thread_info[thread_id].workflow->stats.category_flows[category]++; } } @@ -2921,8 +3010,8 @@ static void dump_realtime_protocol(struct ndpi_workflow * workflow, struct ndpi_ inet_ntop(AF_INET, &flow->src_ip, srcip, sizeof(srcip)); inet_ntop(AF_INET, &flow->dst_ip, dstip, sizeof(dstip)); } else { - snprintf(srcip, sizeof(srcip), "[%s]", flow->src_name); - snprintf(dstip, sizeof(dstip), "[%s]", flow->dst_name); + snprintf(srcip, sizeof(srcip), "[%s]", flow->src_name ? flow->src_name : ""); + snprintf(dstip, sizeof(dstip), "[%s]", flow->dst_name ? flow->dst_name : ""); } ndpi_protocol2name(workflow->ndpi_struct, flow->detected_protocol, app_name, sizeof(app_name)); @@ -2951,7 +3040,6 @@ static void on_protocol_discovered(struct ndpi_workflow * workflow, */ static void setupDetection(u_int16_t thread_id, pcap_t * pcap_handle, struct ndpi_global_context *g_ctx) { - NDPI_PROTOCOL_BITMASK enabled_bitmask; struct ndpi_workflow_prefs prefs; int i, ret; ndpi_cfg_error rc; @@ -2968,10 +3056,8 @@ static void setupDetection(u_int16_t thread_id, pcap_t * pcap_handle, serialization_format, g_ctx); /* Protocols to enable/disable. Default: everything is enabled */ - NDPI_BITMASK_SET_ALL(enabled_bitmask); if(_disabled_protocols != NULL) { - if(parse_proto_name_list(_disabled_protocols, &enabled_bitmask, 1)) - exit(-1); + enable_disable_protocols_list(ndpi_thread_info[thread_id].workflow->ndpi_struct, _disabled_protocols, 1); } if(_categoriesDirPath) { @@ -2984,7 +3070,7 @@ static void setupDetection(u_int16_t thread_id, pcap_t * pcap_handle, if(_domain_suffixes) ndpi_load_domain_suffixes(ndpi_thread_info[thread_id].workflow->ndpi_struct, _domain_suffixes); - + if(_riskyDomainFilePath) ndpi_load_risk_domain_file(ndpi_thread_info[thread_id].workflow->ndpi_struct, _riskyDomainFilePath); @@ -3009,14 +3095,14 @@ static void setupDetection(u_int16_t thread_id, pcap_t * pcap_handle, } } + if(ndpi_thread_info[thread_id].workflow == NULL) + exit(-1); /* Some initialiation functions failed */ + ndpi_thread_info[thread_id].workflow->g_ctx = g_ctx; ndpi_workflow_set_flow_callback(ndpi_thread_info[thread_id].workflow, on_protocol_discovered, NULL); - /* Make sure to load lists before finalizing the initialization */ - ndpi_set_protocol_detection_bitmask2(ndpi_thread_info[thread_id].workflow->ndpi_struct, &enabled_bitmask); - if(_protoFilePath != NULL) ndpi_load_protocols_file(ndpi_thread_info[thread_id].workflow->ndpi_struct, _protoFilePath); @@ -3050,6 +3136,12 @@ static void setupDetection(u_int16_t thread_id, pcap_t * pcap_handle, if(atoi(buf)) monitoring_enabled = 1; } + + unsigned int num_protocols = ndpi_get_num_protocols(ndpi_thread_info[thread_id].workflow->ndpi_struct); + if (!ndpi_stats_init(&ndpi_thread_info[thread_id].workflow->stats, num_protocols)) { + exit(-1); + } + } /* *********************************************** */ @@ -3332,7 +3424,7 @@ static void printFlowsStats() { ndpi_host_ja_fingerprints *newHost = ndpi_malloc(sizeof(ndpi_host_ja_fingerprints)); newHost->host_client_info_hasht = NULL; newHost->host_server_info_hasht = NULL; - newHost->ip_string = all_flows[i].flow->src_name; + newHost->ip_string = all_flows[i].flow->src_name ? all_flows[i].flow->src_name : NULL; newHost->ip = all_flows[i].flow->src_ip; newHost->dns_name = all_flows[i].flow->host_server_name; @@ -3366,7 +3458,7 @@ static void printFlowsStats() { ndpi_ip_dns *newHost = ndpi_malloc(sizeof(ndpi_ip_dns)); newHost->ip = all_flows[i].flow->src_ip; - newHost->ip_string = all_flows[i].flow->src_name; + newHost->ip_string = all_flows[i].flow->src_name ? all_flows[i].flow->src_name : NULL; newHost->dns_name = all_flows[i].flow->host_server_name; ndpi_ja_fingerprints_host *newElement = ndpi_malloc(sizeof(ndpi_ja_fingerprints_host)); @@ -3383,7 +3475,7 @@ static void printFlowsStats() { if(innerElement == NULL) { ndpi_ip_dns *newInnerElement = ndpi_malloc(sizeof(ndpi_ip_dns)); newInnerElement->ip = all_flows[i].flow->src_ip; - newInnerElement->ip_string = all_flows[i].flow->src_name; + newInnerElement->ip_string = all_flows[i].flow->src_name ? all_flows[i].flow->src_name : NULL; newInnerElement->dns_name = all_flows[i].flow->host_server_name; HASH_ADD_INT(hostByJAFound->ipToDNS_ht, ip, newInnerElement); } @@ -3398,7 +3490,7 @@ static void printFlowsStats() { ndpi_host_ja_fingerprints *newHost = ndpi_malloc(sizeof(ndpi_host_ja_fingerprints)); newHost->host_client_info_hasht = NULL; newHost->host_server_info_hasht = NULL; - newHost->ip_string = all_flows[i].flow->dst_name; + newHost->ip_string = all_flows[i].flow->dst_name ? all_flows[i].flow->dst_name : NULL; newHost->ip = all_flows[i].flow->dst_ip; newHost->dns_name = all_flows[i].flow->ssh_tls.server_info; @@ -3429,7 +3521,7 @@ static void printFlowsStats() { ndpi_ip_dns *newHost = ndpi_malloc(sizeof(ndpi_ip_dns)); newHost->ip = all_flows[i].flow->dst_ip; - newHost->ip_string = all_flows[i].flow->dst_name; + newHost->ip_string = all_flows[i].flow->dst_name ? all_flows[i].flow->dst_name : NULL; newHost->dns_name = all_flows[i].flow->ssh_tls.server_info;; ndpi_ja_fingerprints_host *newElement = ndpi_malloc(sizeof(ndpi_ja_fingerprints_host)); @@ -3447,7 +3539,7 @@ static void printFlowsStats() { if(innerElement == NULL) { ndpi_ip_dns *newInnerElement = ndpi_malloc(sizeof(ndpi_ip_dns)); newInnerElement->ip = all_flows[i].flow->dst_ip; - newInnerElement->ip_string = all_flows[i].flow->dst_name; + newInnerElement->ip_string = all_flows[i].flow->dst_name ? all_flows[i].flow->dst_name : NULL; newInnerElement->dns_name = all_flows[i].flow->ssh_tls.server_info; HASH_ADD_INT(hostByJAFound->ipToDNS_ht, ip, newInnerElement); } @@ -3743,6 +3835,11 @@ static void printFlowsStats() { #endif for(i=0; i<num_flows; i++) { +#ifdef PROTO_DEBUG + ndpi_normalize_protocol(ndpi_thread_info[all_flows[i].thread_id].workflow->ndpi_struct, + &all_flows[i].flow->detected_protocol.proto); +#endif + #ifndef DIRECTION_BINS if(enable_doh_dot_detection) { /* Discard flows with few packets per direction */ @@ -3828,9 +3925,9 @@ static void printFlowsStats() { i, ndpi_protocol2name(ndpi_thread_info[0].workflow->ndpi_struct, all_flows[i].flow->detected_protocol, buf, sizeof(buf)), - all_flows[i].flow->src_name, + all_flows[i].flow->src_name ? all_flows[i].flow->src_name : "", ntohs(all_flows[i].flow->src_port), - all_flows[i].flow->dst_name, + all_flows[i].flow->dst_name ? all_flows[i].flow->dst_name : "", ntohs(all_flows[i].flow->dst_port)); print_bin(out, NULL, &bins[i]); @@ -3932,8 +4029,8 @@ static void printFlowsStats() { } } - for(i=0; i<num_flows; i++) - printFlowSerialized(all_flows[i].flow); + for(i=0; i<num_flows; i++) + printFlowSerialized(all_flows[i].flow); } ndpi_free(all_flows); @@ -3953,7 +4050,14 @@ static void printResults(u_int64_t processing_time_usec, u_int64_t setup_time_us long long unsigned int breed_stats_bytes[NUM_BREEDS] = { 0 }; long long unsigned int breed_stats_flows[NUM_BREEDS] = { 0 }; - memset(&cumulative_stats, 0, sizeof(cumulative_stats)); + /* In ndpiReader all the contexts have the same configuration */ + if (!cumulative_stats_initialized) { + unsigned int num_protocols = ndpi_get_num_protocols(ndpi_thread_info[0].workflow->ndpi_struct); + if (!ndpi_stats_init(&cumulative_stats, num_protocols)) { + return; + } + cumulative_stats_initialized = 1; + } for(thread_id = 0; thread_id < num_threads; thread_id++) { if((ndpi_thread_info[thread_id].workflow->stats.total_wire_bytes == 0) @@ -3975,7 +4079,7 @@ static void printResults(u_int64_t processing_time_usec, u_int64_t setup_time_us cumulative_stats.total_ip_bytes += ndpi_thread_info[thread_id].workflow->stats.total_ip_bytes; cumulative_stats.total_discarded_bytes += ndpi_thread_info[thread_id].workflow->stats.total_discarded_bytes; - for(i = 0; i < ndpi_get_num_supported_protocols(ndpi_thread_info[0].workflow->ndpi_struct); i++) { + for (i = 0; i < cumulative_stats.num_protocols; i++) { cumulative_stats.protocol_counter[i] += ndpi_thread_info[thread_id].workflow->stats.protocol_counter[i]; cumulative_stats.protocol_counter_bytes[i] += ndpi_thread_info[thread_id].workflow->stats.protocol_counter_bytes[i]; cumulative_stats.protocol_flows[i] += ndpi_thread_info[thread_id].workflow->stats.protocol_flows[i]; @@ -3985,6 +4089,12 @@ static void printResults(u_int64_t processing_time_usec, u_int64_t setup_time_us cumulative_stats.fpc_protocol_flows[i] += ndpi_thread_info[thread_id].workflow->stats.fpc_protocol_flows[i]; } + for(i = 0; i < NDPI_PROTOCOL_NUM_CATEGORIES; i++) { + cumulative_stats.category_counter[i] += ndpi_thread_info[thread_id].workflow->stats.category_counter[i]; + cumulative_stats.category_counter_bytes[i] += ndpi_thread_info[thread_id].workflow->stats.category_counter_bytes[i]; + cumulative_stats.category_flows[i] += ndpi_thread_info[thread_id].workflow->stats.category_flows[i]; + } + cumulative_stats.ndpi_flow_count += ndpi_thread_info[thread_id].workflow->stats.ndpi_flow_count; cumulative_stats.flow_count[0] += ndpi_thread_info[thread_id].workflow->stats.flow_count[0]; cumulative_stats.flow_count[1] += ndpi_thread_info[thread_id].workflow->stats.flow_count[1]; @@ -4352,7 +4462,7 @@ static void printResults(u_int64_t processing_time_usec, u_int64_t setup_time_us } if(!quiet_mode) printf("\n\nDetected protocols:\n"); - for(i = 0; i <= ndpi_get_num_supported_protocols(ndpi_thread_info[0].workflow->ndpi_struct); i++) { + for(i = 0; i < cumulative_stats.num_protocols; i++) { ndpi_protocol_breed_t breed = ndpi_get_proto_breed(ndpi_thread_info[0].workflow->ndpi_struct, ndpi_map_ndpi_id_to_user_proto_id(ndpi_thread_info[0].workflow->ndpi_struct, i)); @@ -4432,6 +4542,33 @@ static void printResults(u_int64_t processing_time_usec, u_int64_t setup_time_us } } + if(!quiet_mode) { + printf("\n\nCategory statistics:\n"); + + for(i = 0; i < NDPI_PROTOCOL_NUM_CATEGORIES; i++) { + if(cumulative_stats.category_counter[i] > 0) { + printf("\t%-20s packets: %-13llu bytes: %-13llu " + "flows: %-13llu\n", + ndpi_category_get_name(ndpi_thread_info[0].workflow->ndpi_struct, i), + (long long unsigned int)cumulative_stats.category_counter[i], + (long long unsigned int)cumulative_stats.category_counter_bytes[i], + (long long unsigned int)cumulative_stats.category_flows[i]); + } + } + } + if(results_file) { + fprintf(results_file, "\n"); + for(i = 0; i < NDPI_PROTOCOL_NUM_CATEGORIES; i++) { + if(cumulative_stats.category_counter[i] > 0) { + fprintf(results_file, "%-20s %13llu %-13llu %-13llu\n", + ndpi_category_get_name(ndpi_thread_info[0].workflow->ndpi_struct, i), + (long long unsigned int)cumulative_stats.category_counter[i], + (long long unsigned int)cumulative_stats.category_counter_bytes[i], + (long long unsigned int)cumulative_stats.category_flows[i]); + } + } + } + printRiskStats(); printFlowsStats(); @@ -4473,6 +4610,8 @@ static void printResults(u_int64_t processing_time_usec, u_int64_t setup_time_us deletePortsStats(dstStats); dstStats = NULL; } + + ndpi_stats_reset(&cumulative_stats); } /** @@ -4537,7 +4676,7 @@ static int getNextPcapFileFromPlaylist(u_int16_t thread_id, char filename[], u_i static void configurePcapHandle(pcap_t * pcap_handle) { if(!pcap_handle) return; - + if(bpfFilter != NULL) { if(!bpf_cfilter) { if(pcap_compile(pcap_handle, &bpf_code, bpfFilter, 1, 0xFFFFFF00) < 0) { @@ -4546,7 +4685,7 @@ static void configurePcapHandle(pcap_t * pcap_handle) { } bpf_cfilter = &bpf_code; } - + if(pcap_setfilter(pcap_handle, bpf_cfilter) < 0) { printf("pcap_setfilter error: '%s'\n", pcap_geterr(pcap_handle)); } else { @@ -4758,22 +4897,6 @@ static void ndpi_process_packet(u_char *args, tot_len += 4 + htons(tlv->length); tlv = (struct ndpi_packet_tlv *)&trailer->metadata[tot_len]; } - if(flow->ssh_tls.obfuscated_heur_matching_set.pkts[0] != 0) { - tlv->type = ntohs(WIRESHARK_METADATA_TLS_HEURISTICS_MATCHING_FINGERPRINT); - tlv->length = ntohs(sizeof(struct ndpi_tls_obfuscated_heuristic_matching_set)); - struct ndpi_tls_obfuscated_heuristic_matching_set *s = (struct ndpi_tls_obfuscated_heuristic_matching_set *)tlv->data; - s->bytes[0] = ntohl(flow->ssh_tls.obfuscated_heur_matching_set.bytes[0]); - s->bytes[1] = ntohl(flow->ssh_tls.obfuscated_heur_matching_set.bytes[1]); - s->bytes[2] = ntohl(flow->ssh_tls.obfuscated_heur_matching_set.bytes[2]); - s->bytes[3] = ntohl(flow->ssh_tls.obfuscated_heur_matching_set.bytes[3]); - s->pkts[0] = ntohl(flow->ssh_tls.obfuscated_heur_matching_set.pkts[0]); - s->pkts[1] = ntohl(flow->ssh_tls.obfuscated_heur_matching_set.pkts[1]); - s->pkts[2] = ntohl(flow->ssh_tls.obfuscated_heur_matching_set.pkts[2]); - s->pkts[3] = ntohl(flow->ssh_tls.obfuscated_heur_matching_set.pkts[3]); - /* TODO: boundary check */ - tot_len += 4 + htons(tlv->length); - tlv = (struct ndpi_packet_tlv *)&trailer->metadata[tot_len]; - } flow->detection_completed = 2; /* Avoid exporting metadata again. If we really want to have the metadata on Wireshark for *all* @@ -4816,9 +4939,8 @@ static void ndpi_process_packet(u_char *args, for(i=0; i<ndpi_thread_info[thread_id].workflow->prefs.num_roots; i++) { ndpi_tdestroy(ndpi_thread_info[thread_id].workflow->ndpi_flows_root[i], ndpi_flow_info_freer); ndpi_thread_info[thread_id].workflow->ndpi_flows_root[i] = NULL; - - memset(&ndpi_thread_info[thread_id].workflow->stats, 0, sizeof(struct ndpi_stats)); } + ndpi_stats_reset(&ndpi_thread_info[thread_id].workflow->stats); if(!quiet_mode) printf("\n-------------------------------------------\n\n"); @@ -5173,26 +5295,20 @@ static void dgaUnitTest() { NULL }; int debug = 0, i; - NDPI_PROTOCOL_BITMASK all; struct ndpi_detection_module_struct *ndpi_str = ndpi_init_detection_module(NULL); assert(ndpi_str != NULL); - NDPI_BITMASK_SET_ALL(all); - ndpi_set_protocol_detection_bitmask2(ndpi_str, &all); - - ndpi_finalize_initialization(ndpi_str); - - assert(ndpi_str != NULL); + assert(ndpi_finalize_initialization(ndpi_str) ==0); for(i=0; non_dga[i] != NULL; i++) { if(debug) printf("Checking non DGA %s\n", non_dga[i]); - assert(ndpi_check_dga_name(ndpi_str, NULL, (char*)non_dga[i], 1, 1) == 0); + assert(ndpi_check_dga_name(ndpi_str, NULL, (char*)non_dga[i], 1, 1, 0) == 0); } for(i=0; dga[i] != NULL; i++) { if(debug) printf("Checking DGA %s\n", non_dga[i]); - assert(ndpi_check_dga_name(ndpi_str, NULL, (char*)dga[i], 1, 1) == 1); + assert(ndpi_check_dga_name(ndpi_str, NULL, (char*)dga[i], 1, 1, 0) == 1); } ndpi_exit_detection_module(ndpi_str); @@ -6125,6 +6241,60 @@ void memmemUnitTest(void) { /* *********************************************** */ +void memcasecmpUnitTest(void) +{ + /* Test 1: NULL pointers */ + assert(ndpi_memcasecmp(NULL, NULL, 5) == 0); + assert(ndpi_memcasecmp(NULL, "string", 6) == -1); + assert(ndpi_memcasecmp("string", NULL, 6) == 1); + + /* Test 2: Zero length */ + assert(ndpi_memcasecmp("string", "different", 0) == 0); + + /* Test 3: Single byte comparison */ + assert(ndpi_memcasecmp("a", "a", 1) == 0); + assert(ndpi_memcasecmp("a", "A", 1) == 0); + assert(ndpi_memcasecmp("a", "b", 1) < 0); + assert(ndpi_memcasecmp("b", "a", 1) > 0); + + /* Test 4: Case insensitivity */ + assert(ndpi_memcasecmp("STRING", "string", 6) == 0); + assert(ndpi_memcasecmp("String", "sTrInG", 6) == 0); + + /* Test 5: Various string comparisons */ + assert(ndpi_memcasecmp("string", "string", 6) == 0); + assert(ndpi_memcasecmp("string", "strong", 6) < 0); + assert(ndpi_memcasecmp("strong", "string", 6) > 0); + assert(ndpi_memcasecmp("abc", "abcd", 3) == 0); + assert(ndpi_memcasecmp("abcd", "abc", 3) == 0); + + /* Test 6: Optimization for checking first and last bytes */ + assert(ndpi_memcasecmp("aBc", "abc", 3) == 0); + assert(ndpi_memcasecmp("abc", "abC", 3) == 0); + assert(ndpi_memcasecmp("abc", "def", 3) < 0); + assert(ndpi_memcasecmp("abz", "abx", 3) > 0); + assert(ndpi_memcasecmp("axc", "ayc", 3) < 0); + + /* Test 7: Edge cases with non-printable characters and embedded zeros */ + const char str1[] = {0, 'a', 'b', 'c'}; + const char str2[] = {0, 'a', 'b', 'c'}; + assert(ndpi_memcasecmp(str1, str2, 4) == 0); + + const char str3[] = {0, 'a', 'b', 'c'}; + const char str4[] = {1, 'a', 'b', 'c'}; + assert(ndpi_memcasecmp(str3, str4, 4) < 0); + + const char str5[] = {'a', 'b', 'c', 0}; + const char str6[] = {'a', 'b', 'c', 1}; + assert(ndpi_memcasecmp(str5, str6, 4) < 0); + + const char str7[] = {'a', 'b', 0, 'd'}; + const char str8[] = {'a', 'b', 1, 'd'}; + assert(ndpi_memcasecmp(str7, str8, 4) < 0); +} + +/* *********************************************** */ + void mahalanobisUnitTest() { /* Example based on: https://supplychenmanagement.com/2019/03/06/calculating-mahalanobis-distance/ */ @@ -6144,6 +6314,36 @@ void mahalanobisUnitTest() /* *********************************************** */ +void bitmaskUnitTest() +{ + struct ndpi_bitmask b; + int i; + + assert(ndpi_bitmask_alloc(&b, 512) == 0); + for(i = 0; i < b.max_bits; i++) { + ndpi_bitmask_set(&b, i); + assert(ndpi_bitmask_is_set(&b, i)); + } + for(i = 0; i < b.max_bits; i++) { + ndpi_bitmask_clear(&b, i); + assert(!ndpi_bitmask_is_set(&b, i)); + } + ndpi_bitmask_set_all(&b); + for(i = 0; i < b.max_bits; i++) + assert(ndpi_bitmask_is_set(&b, i)); + ndpi_bitmask_reset(&b); + for(i = 0; i < b.max_bits; i++) + assert(!ndpi_bitmask_is_set(&b, i)); + for(i = 0; i < b.max_bits; i++) { + ndpi_bitmask_set(&b, i); + assert(ndpi_bitmask_is_set(&b, i)); + } + + ndpi_bitmask_free(&b); +} + +/* *********************************************** */ + void filterUnitTest() { ndpi_filter* f = ndpi_filter_alloc(); u_int32_t v, i; @@ -6283,17 +6483,13 @@ void outlierUnitTest() { void loadStressTest() { struct ndpi_detection_module_struct *ndpi_struct_shadow = ndpi_init_detection_module(NULL); - NDPI_PROTOCOL_BITMASK all; if(ndpi_struct_shadow) { int i; - NDPI_BITMASK_SET_ALL(all); - ndpi_set_protocol_detection_bitmask2(ndpi_struct_shadow, &all); - for(i=1; i<100000; i++) { char name[32]; - ndpi_protocol_category_t id = CUSTOM_CATEGORY_MALWARE; + ndpi_protocol_category_t id = NDPI_PROTOCOL_CATEGORY_MALWARE; u_int8_t value = (u_int8_t)i; snprintf(name, sizeof(name), "%d.com", i); @@ -6397,7 +6593,7 @@ void ballTreeUnitTest() { /* *********************************************** */ void cryptDecryptUnitTest() { - u_char enc_dec_key[64] = "9dedb817e5a8805c1de62eb8982665b9a2b4715174c34d23b9a46ffafacfb2a7" /* SHA256("nDPI") */; + u_char enc_dec_key[] = "9dedb817e5a8805c1de62eb8982665b9a2b4715174c34d23b9a46ffafacfb2a7" /* SHA256("nDPI") */; const char *test_string = "The quick brown fox jumps over the lazy dog"; char *enc, *dec; u_int16_t e_len, d_len, t_len = strlen(test_string); @@ -6417,7 +6613,6 @@ void cryptDecryptUnitTest() { /* *********************************************** */ void encodeDomainsUnitTest() { - NDPI_PROTOCOL_BITMASK all; struct ndpi_detection_module_struct *ndpi_str = ndpi_init_detection_module(NULL); const char *lists_path = "../lists/public_suffix_list.dat"; struct stat st; @@ -6428,9 +6623,6 @@ void encodeDomainsUnitTest() { char *str; ndpi_protocol_category_t id; - NDPI_BITMASK_SET_ALL(all); - ndpi_set_protocol_detection_bitmask2(ndpi_str, &all); - assert(ndpi_load_domain_suffixes(ndpi_str, (char*)lists_path) == 0); ndpi_get_host_domain_suffix(ndpi_str, "lcb.it", &suffix_id); @@ -6449,7 +6641,19 @@ void encodeDomainsUnitTest() { str = (char*)"222.0grand-casino.com"; assert(ndpi_get_custom_category_match(ndpi_str, str, strlen(str), &id) == 0); assert(id == 107); str = (char*)"10bet.com"; assert(ndpi_get_custom_category_match(ndpi_str, str, strlen(str), &id) == 0); assert(id == 107); str = (char*)"www.ntop.org"; assert(ndpi_get_custom_category_match(ndpi_str, str, strlen(str), &id) == -1); assert(id == 0); - str = (char*)"www.andrewpope.com"; assert(ndpi_get_custom_category_match(ndpi_str, str, strlen(str), &id) == 0); assert(id == 100); + str = (char*)"lifyqyi.com"; assert(ndpi_get_custom_category_match(ndpi_str, str, strlen(str), &id) == 0); assert(id == 100); + } + + ndpi_exit_detection_module(ndpi_str); +} + +/* *********************************************** */ + +void checkProtocolIDsUnitTest() { + struct ndpi_detection_module_struct *ndpi_str = ndpi_init_detection_module(NULL); + + if(ndpi_str != NULL) { + assert(ndpi_load_protocols_dir(ndpi_str, "../lists/protocols")); } ndpi_exit_detection_module(ndpi_str); @@ -6458,7 +6662,6 @@ void encodeDomainsUnitTest() { /* *********************************************** */ void domainsUnitTest() { - NDPI_PROTOCOL_BITMASK all; struct ndpi_detection_module_struct *ndpi_str = ndpi_init_detection_module(NULL); const char *lists_path = "../lists/public_suffix_list.dat"; struct stat st; @@ -6466,11 +6669,11 @@ void domainsUnitTest() { if(stat(lists_path, &st) == 0) { u_int16_t suffix_id; - NDPI_BITMASK_SET_ALL(all); - ndpi_set_protocol_detection_bitmask2(ndpi_str, &all); - assert(ndpi_load_domain_suffixes(ndpi_str, (char*)lists_path) == 0); + assert(strcmp(ndpi_get_host_domain(ndpi_str, "1.0.0.127.in-addr.arpa"), "in-addr.arpa") == 0); + assert(strcmp(ndpi_get_host_domain(ndpi_str, "fe80::fd:5447:b2d1:40e0"), "fe80::fd:5447:b2d1:40e0") == 0); + assert(strcmp(ndpi_get_host_domain(ndpi_str, "192.168.1.2"), "192.168.1.2") == 0); assert(strcmp(ndpi_get_host_domain(ndpi_str, "extension.femetrics.grammarly.io"), "grammarly.io") == 0); assert(strcmp(ndpi_get_host_domain(ndpi_str, "www.ovh.commander1.com"), "commander1.com") == 0); @@ -6497,14 +6700,11 @@ void domainSearchUnitTest() { u_int16_t class_id; struct ndpi_detection_module_struct *ndpi_str = ndpi_init_detection_module(NULL); u_int8_t trace = 0; - NDPI_PROTOCOL_BITMASK all; assert(ndpi_str); assert(sc); - NDPI_BITMASK_SET_ALL(all); - ndpi_set_protocol_detection_bitmask2(ndpi_str, &all); - ndpi_finalize_initialization(ndpi_str); + assert(ndpi_finalize_initialization(ndpi_str) == 0); ndpi_domain_classify_add(ndpi_str, sc, NDPI_PROTOCOL_NTOP, ".ntop.org"); ndpi_domain_classify_add(ndpi_str, sc, NDPI_PROTOCOL_NTOP, domain); @@ -6533,14 +6733,11 @@ void domainSearchUnitTest2() { struct ndpi_detection_module_struct *ndpi_str = ndpi_init_detection_module(NULL); ndpi_domain_classify *c = ndpi_domain_classify_alloc(); u_int16_t class_id = 9; - NDPI_PROTOCOL_BITMASK all; assert(ndpi_str); assert(c); - NDPI_BITMASK_SET_ALL(all); - ndpi_set_protocol_detection_bitmask2(ndpi_str, &all); - ndpi_finalize_initialization(ndpi_str); + assert(ndpi_finalize_initialization(ndpi_str) == 0); ndpi_domain_classify_add(ndpi_str, c, class_id, "ntop.org"); ndpi_domain_classify_add(ndpi_str, c, class_id, "apple.com"); @@ -6605,6 +6802,9 @@ int main(int argc, char **argv) { int skip_unit_tests = 1; #endif + + //checkProtocolIDsUnitTest(); return(0); + #ifdef DEBUG_TRACE trace = fopen("/tmp/ndpiReader.log", "a"); @@ -6667,7 +6867,6 @@ int main(int argc, char **argv) { automataUnitTest(); automataDomainsUnitTest(); analyzeUnitTest(); - ndpi_self_check_host_match(stderr); analysisUnitTest(); compressedBitmapUnitTest(); strtonumUnitTest(); @@ -6675,7 +6874,9 @@ int main(int argc, char **argv) { strnstrUnitTest(); strncasestrUnitTest(); memmemUnitTest(); + memcasecmpUnitTest(); mahalanobisUnitTest(); + bitmaskUnitTest(); #endif } @@ -6691,7 +6892,7 @@ int main(int argc, char **argv) { ndpiCheckHostStringMatch(domain_to_check); exit(0); } - + if(ip_port_to_check) { ndpiCheckIPMatch(ip_port_to_check); exit(0); @@ -6708,7 +6909,7 @@ int main(int argc, char **argv) { #ifdef CUSTOM_NDPI_PROTOCOLS #include "../../nDPI-custom/ndpiReader_init.c" #endif - + if(!quiet_mode) { printf("\n-----------------------------------------------------------\n" "* NOTE: This is demo app to show *some* nDPI features.\n" |