diff options
author | Luca Deri <deri@ntop.org> | 2019-11-17 17:51:45 +0100 |
---|---|---|
committer | Luca Deri <deri@ntop.org> | 2019-11-17 17:51:45 +0100 |
commit | 51cfdfb0d80a7bbcc11bc3b95d1696d8dae900c2 (patch) | |
tree | d842aa211a58677ae596ca0cea68a622d7152d03 /example/ndpiReader.c | |
parent | 53108c9ff15aa52db76ad271db256ff7070efcff (diff) |
Removed unused JSON-C code
Diffstat (limited to 'example/ndpiReader.c')
-rw-r--r-- | example/ndpiReader.c | 1329 |
1 files changed, 143 insertions, 1186 deletions
diff --git a/example/ndpiReader.c b/example/ndpiReader.c index abf1bce54..84f7ddb72 100644 --- a/example/ndpiReader.c +++ b/example/ndpiReader.c @@ -52,10 +52,6 @@ #include <sys/mman.h> #include <libgen.h> -#ifdef HAVE_JSON_C -#include <json.h> -#endif - #include "reader_util.h" @@ -68,26 +64,14 @@ static char *results_path = NULL; static char * bpfFilter = NULL; /**< bpf filter */ static char *_protoFilePath = NULL; /**< Protocol file path */ static char *_customCategoryFilePath= NULL; /**< Custom categories file path */ -#ifdef HAVE_JSON_C -static char *_statsFilePath = NULL; /**< Top stats file path */ -static char *_diagnoseFilePath = NULL; /**< Top stats file path */ -static char *_jsonFilePath = NULL; /**< JSON file path */ -static FILE *stats_fp = NULL; /**< for Top Stats JSON file */ -static json_object *jArray_known_flows = NULL, *jArray_unknown_flows = NULL; -static json_object *jArray_topStats = NULL; -#endif static FILE *csv_fp = NULL; /**< for CSV export */ static u_int8_t live_capture = 0; static u_int8_t undetected_flows_deleted = 0; /** User preferences **/ u_int8_t enable_protocol_guess = 1, enable_payload_analyzer = 0; -u_int8_t verbose = 0, json_flag = 0, enable_joy_stats = 0; +u_int8_t verbose = 0, enable_joy_stats = 0; int nDPI_LogLevel = 0; char *_debug_protocols = NULL; -static u_int8_t stats_flag = 0, bpf_filter_flag = 0; -#ifdef HAVE_JSON_C -static u_int8_t file_first_time = 1; -#endif u_int8_t human_readeable_string_len = 5; u_int8_t max_num_udp_dissected_pkts = 16 /* 8 is enough for most protocols, Signal requires more */, max_num_tcp_dissected_pkts = 80 /* due to telnet */; static u_int32_t pcap_analysis_duration = (u_int32_t)-1; @@ -356,7 +340,6 @@ static void help(u_int long_help) { " -l <num loops> | Number of detection loops (test only)\n" " -n <num threads> | Number of threads. Default: number of interfaces in -i.\n" " | Ignored with pcap files.\n" - " -j <file.json> | Specify a file to write the content of packets in .json format\n" #ifdef linux " -g <id:id...> | Thread affinity mask (one core id per thread)\n" #endif @@ -386,9 +369,6 @@ static void help(u_int long_help) { " -V <1-4> | nDPI logging level\n" " | 1 - trace, 2 - debug, 3 - full debug\n" " | >3 - full debug + dbg_proto = all\n" - " -b <file.json> | Specify a file to write port based diagnose statistics\n" - " -x <file.json> | Produce bpf filters for specified diagnose file. Use\n" - " | this option only for .json files generated with -b flag.\n" " -T <num> | Max number of TCP processed packets before giving up [default: %u]\n" " -U <num> | Max number of UDP processed packets before giving up [default: %u]\n" , @@ -458,7 +438,6 @@ static struct option longopts[] = { { "verbose", no_argument, NULL, 'v'}, { "version", no_argument, NULL, 'V'}, { "help", no_argument, NULL, 'h'}, - { "json", required_argument, NULL, 'j'}, { "joy", required_argument, NULL, 'J'}, { "payload-analysis", required_argument, NULL, 'P'}, { "result-path", required_argument, NULL, 'w'}, @@ -637,7 +616,7 @@ static void parseOptions(int argc, char **argv) { } #endif - while((opt = getopt_long(argc, argv, "e:c:C:df:g:i:hp:P:l:s:tv:V:n:j:Jrp:w:q0123:456:7:89:m:b:x:T:U:", + while((opt = getopt_long(argc, argv, "e:c:C:df:g:i:hp:P:l:s:tv:V:n:Jrp:w:q0123:456:7:89:m:T:U:", longopts, &option_idx)) != EOF) { #ifdef DEBUG_TRACE if(trace) fprintf(trace, " #### -%c [%s] #### \n", opt, optarg ? optarg : ""); @@ -657,28 +636,10 @@ static void parseOptions(int argc, char **argv) { _pcap_file[0] = optarg; break; - case 'b': -#ifndef HAVE_JSON_C - printf("WARNING: this copy of ndpiReader has been compiled without JSON-C: json export disabled\n"); -#else - _statsFilePath = optarg; - stats_flag = 1; -#endif - break; - case 'm': pcap_analysis_duration = atol(optarg); break; - case 'x': -#ifndef HAVE_JSON_C - printf("WARNING: this copy of ndpiReader has been compiled without JSON-C: json export disabled\n"); -#else - _diagnoseFilePath = optarg; - bpf_filter_flag = 1; -#endif - break; - case 'f': case '6': bpfFilter = optarg; @@ -772,15 +733,6 @@ static void parseOptions(int argc, char **argv) { } break; - case 'j': -#ifndef HAVE_JSON_C - printf("WARNING: this copy of ndpiReader has been compiled without json-c: JSON export disabled\n"); -#else - _jsonFilePath = optarg; - json_flag = 1; -#endif - break; - case 'w': results_path = strdup(optarg); if((results_file = fopen(results_path, "w")) == NULL) { @@ -850,17 +802,6 @@ static void parseOptions(int argc, char **argv) { } #ifndef USE_DPDK - if(!bpf_filter_flag) { - if(do_capture) { - quiet_mode = 1; - extcap_capture(); - } - - // check parameters - if(!bpf_filter_flag && (_pcap_file[0] == NULL || strcmp(_pcap_file[0], "") == 0)) { - help(0); - } - if(strchr(_pcap_file[0], ',')) { /* multiple ingress interfaces */ num_threads = 0; /* setting number of threads = number of interfaces */ __pcap_file = strtok(_pcap_file[0], ","); @@ -887,7 +828,6 @@ static void parseOptions(int argc, char **argv) { } } #endif - } #endif #ifdef DEBUG_TRACE @@ -1019,12 +959,10 @@ char* printUrlRisk(ndpi_url_risk risk) { * @brief Print the flow */ static void printFlow(u_int16_t id, struct ndpi_flow_info *flow, u_int16_t thread_id) { -#ifdef HAVE_JSON_C - json_object *jObj; -#endif FILE *out = results_file ? results_file : stdout; u_int8_t known_tls; char buf[32], buf1[64]; + u_int i; if(csv_fp != NULL) { float data_ratio = ndpi_data_ratio(flow->src2dst_bytes, flow->dst2src_bytes); @@ -1038,12 +976,12 @@ static void printFlow(u_int16_t id, struct ndpi_flow_info *flow, u_int16_t threa f/1000.0, l/1000.0, flow->src_name, ntohs(flow->src_port), flow->dst_name, ntohs(flow->dst_port) - ); + ); fprintf(csv_fp, "%u.%u,%s,", - flow->detected_protocol.master_protocol, flow->detected_protocol.app_protocol, - ndpi_protocol2name(ndpi_thread_info[thread_id].workflow->ndpi_struct, - flow->detected_protocol, buf, sizeof(buf))); + flow->detected_protocol.master_protocol, flow->detected_protocol.app_protocol, + ndpi_protocol2name(ndpi_thread_info[thread_id].workflow->ndpi_struct, + flow->detected_protocol, buf, sizeof(buf))); fprintf(csv_fp, "%u,%llu,", flow->src2dst_packets, (long long unsigned int) flow->src2dst_bytes); fprintf(csv_fp, "%u,%llu,", flow->dst2src_packets, (long long unsigned int) flow->dst2src_bytes); @@ -1054,13 +992,13 @@ static void printFlow(u_int16_t id, struct ndpi_flow_info *flow, u_int16_t threa ndpi_data_min(flow->iat_flow), ndpi_data_average(flow->iat_flow), ndpi_data_max(flow->iat_flow), ndpi_data_stddev(flow->iat_flow)); fprintf(csv_fp, "%u,%.1f,%u,%.1f,%u,%.1f,%u,%.1f,", - ndpi_data_min(flow->iat_c_to_s), ndpi_data_average(flow->iat_c_to_s), ndpi_data_max(flow->iat_c_to_s), ndpi_data_stddev(flow->iat_c_to_s), - ndpi_data_min(flow->iat_s_to_c), ndpi_data_average(flow->iat_s_to_c), ndpi_data_max(flow->iat_s_to_c), ndpi_data_stddev(flow->iat_s_to_c)); + ndpi_data_min(flow->iat_c_to_s), ndpi_data_average(flow->iat_c_to_s), ndpi_data_max(flow->iat_c_to_s), ndpi_data_stddev(flow->iat_c_to_s), + ndpi_data_min(flow->iat_s_to_c), ndpi_data_average(flow->iat_s_to_c), ndpi_data_max(flow->iat_s_to_c), ndpi_data_stddev(flow->iat_s_to_c)); /* Packet Length */ fprintf(csv_fp, "%u,%.1f,%u,%.1f,%u,%.1f,%u,%.1f,", - ndpi_data_min(flow->pktlen_c_to_s), ndpi_data_average(flow->pktlen_c_to_s), ndpi_data_max(flow->pktlen_c_to_s), ndpi_data_stddev(flow->pktlen_c_to_s), - ndpi_data_min(flow->pktlen_s_to_c), ndpi_data_average(flow->pktlen_s_to_c), ndpi_data_max(flow->pktlen_s_to_c), ndpi_data_stddev(flow->pktlen_s_to_c)); + ndpi_data_min(flow->pktlen_c_to_s), ndpi_data_average(flow->pktlen_c_to_s), ndpi_data_max(flow->pktlen_c_to_s), ndpi_data_stddev(flow->pktlen_c_to_s), + ndpi_data_min(flow->pktlen_s_to_c), ndpi_data_average(flow->pktlen_s_to_c), ndpi_data_max(flow->pktlen_s_to_c), ndpi_data_stddev(flow->pktlen_s_to_c)); fprintf(csv_fp, "%s,%s,", (flow->ssh_tls.client_info[0] != '\0') ? flow->ssh_tls.client_info : "", @@ -1080,210 +1018,141 @@ static void printFlow(u_int16_t id, struct ndpi_flow_info *flow, u_int16_t threa (flow->ssh_tls.server_hassh[0] != '\0') ? flow->ssh_tls.server_hassh : "" ); - fprintf(csv_fp, "\n"); + fprintf(csv_fp, "\n"); } if((verbose != 1) && (verbose != 2)) return; - if(!json_flag) { - u_int i; - #if 1 - fprintf(out, "\t%u", id); + fprintf(out, "\t%u", id); #else - fprintf(out, "\t%u(%u)", id, flow->flow_id); + fprintf(out, "\t%u(%u)", id, flow->flow_id); #endif - fprintf(out, "\t%s ", ipProto2Name(flow->protocol)); - - fprintf(out, "%s%s%s:%u %s %s%s%s:%u ", - (flow->ip_version == 6) ? "[" : "", - flow->src_name, (flow->ip_version == 6) ? "]" : "", ntohs(flow->src_port), - flow->bidirectional ? "<->" : "->", - (flow->ip_version == 6) ? "[" : "", - flow->dst_name, (flow->ip_version == 6) ? "]" : "", ntohs(flow->dst_port) - ); - - if(flow->vlan_id > 0) fprintf(out, "[VLAN: %u]", flow->vlan_id); - if(enable_payload_analyzer) fprintf(out, "[flowId: %u]", flow->flow_id); - - if(enable_joy_stats) { - /* Print entropy values for monitored flows. */ - flowGetBDMeanandVariance(flow); - fflush(out); - fprintf(out, "[score: %.4f]", flow->entropy.score); - } + fprintf(out, "\t%s ", ipProto2Name(flow->protocol)); - fprintf(out, "[proto: "); - if(flow->tunnel_type != ndpi_no_tunnel) - fprintf(out, "%s:", ndpi_tunnel2str(flow->tunnel_type)); - - fprintf(out, "%s/%s]", - ndpi_protocol2id(ndpi_thread_info[thread_id].workflow->ndpi_struct, - flow->detected_protocol, buf, sizeof(buf)), - ndpi_protocol2name(ndpi_thread_info[thread_id].workflow->ndpi_struct, - flow->detected_protocol, buf1, sizeof(buf1))); - - if(flow->detected_protocol.category != 0) - fprintf(out, "[cat: %s/%u]", - ndpi_category_get_name(ndpi_thread_info[thread_id].workflow->ndpi_struct, - flow->detected_protocol.category), - (unsigned int)flow->detected_protocol.category); - - fprintf(out, "[%u pkts/%llu bytes ", flow->src2dst_packets, (long long unsigned int) flow->src2dst_bytes); - fprintf(out, "%s %u pkts/%llu bytes]", - (flow->dst2src_packets > 0) ? "<->" : "->", - flow->dst2src_packets, (long long unsigned int) flow->dst2src_bytes); - - if(flow->telnet.username[0] != '\0') fprintf(out, "[Username: %s]", flow->telnet.username); - if(flow->host_server_name[0] != '\0') fprintf(out, "[Host: %s]", flow->host_server_name); - - if(flow->info[0] != '\0') fprintf(out, "[%s]", flow->info); - - if((flow->src2dst_packets+flow->dst2src_packets) > 5) { - if(flow->iat_c_to_s && flow->iat_s_to_c) { - float data_ratio = ndpi_data_ratio(flow->src2dst_bytes, flow->dst2src_bytes); - - fprintf(out, "[bytes ratio: %.3f (%s)]", data_ratio, ndpi_data_ratio2str(data_ratio)); - - /* IAT (Inter Arrival Time) */ - fprintf(out, "[IAT c2s/s2c min/avg/max/stddev: %u/%u %.1f/%.1f %u/%u %.1f/%.1f]", - ndpi_data_min(flow->iat_c_to_s), ndpi_data_min(flow->iat_s_to_c), - (float)ndpi_data_average(flow->iat_c_to_s), (float)ndpi_data_average(flow->iat_s_to_c), - ndpi_data_max(flow->iat_c_to_s), ndpi_data_max(flow->iat_s_to_c), - (float)ndpi_data_stddev(flow->iat_c_to_s), (float)ndpi_data_stddev(flow->iat_s_to_c)); - - /* Packet Length */ - fprintf(out, "[Pkt Len c2s/s2c min/avg/max/stddev: %u/%u %.1f/%.1f %u/%u %.1f/%.1f]", - ndpi_data_min(flow->pktlen_c_to_s), ndpi_data_min(flow->pktlen_s_to_c), - ndpi_data_average(flow->pktlen_c_to_s), ndpi_data_average(flow->pktlen_s_to_c), - ndpi_data_max(flow->pktlen_c_to_s), ndpi_data_max(flow->pktlen_s_to_c), - ndpi_data_stddev(flow->pktlen_c_to_s), ndpi_data_stddev(flow->pktlen_s_to_c)); - } - } + fprintf(out, "%s%s%s:%u %s %s%s%s:%u ", + (flow->ip_version == 6) ? "[" : "", + flow->src_name, (flow->ip_version == 6) ? "]" : "", ntohs(flow->src_port), + flow->bidirectional ? "<->" : "->", + (flow->ip_version == 6) ? "[" : "", + flow->dst_name, (flow->ip_version == 6) ? "]" : "", ntohs(flow->dst_port) + ); + + if(flow->vlan_id > 0) fprintf(out, "[VLAN: %u]", flow->vlan_id); + if(enable_payload_analyzer) fprintf(out, "[flowId: %u]", flow->flow_id); - if(flow->http.url[0] != '\0') - fprintf(out, "[URL: %s%s][StatusCode: %u][ContentType: %s][UserAgent: %s]", - flow->http.url, - printUrlRisk(ndpi_validate_url(flow->http.url)), - flow->http.response_status_code, - flow->http.content_type, flow->http.user_agent); + if(enable_joy_stats) { + /* Print entropy values for monitored flows. */ + flowGetBDMeanandVariance(flow); + fflush(out); + fprintf(out, "[score: %.4f]", flow->entropy.score); + } + + fprintf(out, "[proto: "); + if(flow->tunnel_type != ndpi_no_tunnel) + fprintf(out, "%s:", ndpi_tunnel2str(flow->tunnel_type)); + + fprintf(out, "%s/%s]", + ndpi_protocol2id(ndpi_thread_info[thread_id].workflow->ndpi_struct, + flow->detected_protocol, buf, sizeof(buf)), + ndpi_protocol2name(ndpi_thread_info[thread_id].workflow->ndpi_struct, + flow->detected_protocol, buf1, sizeof(buf1))); + + if(flow->detected_protocol.category != 0) + fprintf(out, "[cat: %s/%u]", + ndpi_category_get_name(ndpi_thread_info[thread_id].workflow->ndpi_struct, + flow->detected_protocol.category), + (unsigned int)flow->detected_protocol.category); + + fprintf(out, "[%u pkts/%llu bytes ", flow->src2dst_packets, (long long unsigned int) flow->src2dst_bytes); + fprintf(out, "%s %u pkts/%llu bytes]", + (flow->dst2src_packets > 0) ? "<->" : "->", + flow->dst2src_packets, (long long unsigned int) flow->dst2src_bytes); + + if(flow->telnet.username[0] != '\0') fprintf(out, "[Username: %s]", flow->telnet.username); + if(flow->host_server_name[0] != '\0') fprintf(out, "[Host: %s]", flow->host_server_name); + + if(flow->info[0] != '\0') fprintf(out, "[%s]", flow->info); + + if((flow->src2dst_packets+flow->dst2src_packets) > 5) { + if(flow->iat_c_to_s && flow->iat_s_to_c) { + float data_ratio = ndpi_data_ratio(flow->src2dst_bytes, flow->dst2src_bytes); + + fprintf(out, "[bytes ratio: %.3f (%s)]", data_ratio, ndpi_data_ratio2str(data_ratio)); + + /* IAT (Inter Arrival Time) */ + fprintf(out, "[IAT c2s/s2c min/avg/max/stddev: %u/%u %.1f/%.1f %u/%u %.1f/%.1f]", + ndpi_data_min(flow->iat_c_to_s), ndpi_data_min(flow->iat_s_to_c), + (float)ndpi_data_average(flow->iat_c_to_s), (float)ndpi_data_average(flow->iat_s_to_c), + ndpi_data_max(flow->iat_c_to_s), ndpi_data_max(flow->iat_s_to_c), + (float)ndpi_data_stddev(flow->iat_c_to_s), (float)ndpi_data_stddev(flow->iat_s_to_c)); + + /* Packet Length */ + fprintf(out, "[Pkt Len c2s/s2c min/avg/max/stddev: %u/%u %.1f/%.1f %u/%u %.1f/%.1f]", + ndpi_data_min(flow->pktlen_c_to_s), ndpi_data_min(flow->pktlen_s_to_c), + ndpi_data_average(flow->pktlen_c_to_s), ndpi_data_average(flow->pktlen_s_to_c), + ndpi_data_max(flow->pktlen_c_to_s), ndpi_data_max(flow->pktlen_s_to_c), + ndpi_data_stddev(flow->pktlen_c_to_s), ndpi_data_stddev(flow->pktlen_s_to_c)); + } + } + + if(flow->http.url[0] != '\0') + fprintf(out, "[URL: %s%s][StatusCode: %u][ContentType: %s][UserAgent: %s]", + flow->http.url, + printUrlRisk(ndpi_validate_url(flow->http.url)), + flow->http.response_status_code, + flow->http.content_type, flow->http.user_agent); - if(flow->ssh_tls.ssl_version != 0) fprintf(out, "[%s]", ndpi_ssl_version2str(flow->ssh_tls.ssl_version, &known_tls)); - if(flow->ssh_tls.client_info[0] != '\0') fprintf(out, "[Client: %s]", flow->ssh_tls.client_info); - if(flow->ssh_tls.client_hassh[0] != '\0') fprintf(out, "[HASSH-C: %s]", flow->ssh_tls.client_hassh); + if(flow->ssh_tls.ssl_version != 0) fprintf(out, "[%s]", ndpi_ssl_version2str(flow->ssh_tls.ssl_version, &known_tls)); + if(flow->ssh_tls.client_info[0] != '\0') fprintf(out, "[Client: %s]", flow->ssh_tls.client_info); + if(flow->ssh_tls.client_hassh[0] != '\0') fprintf(out, "[HASSH-C: %s]", flow->ssh_tls.client_hassh); - if(flow->ssh_tls.ja3_client[0] != '\0') fprintf(out, "[JA3C: %s%s]", flow->ssh_tls.ja3_client, - print_cipher(flow->ssh_tls.client_unsafe_cipher)); + if(flow->ssh_tls.ja3_client[0] != '\0') fprintf(out, "[JA3C: %s%s]", flow->ssh_tls.ja3_client, + print_cipher(flow->ssh_tls.client_unsafe_cipher)); - if(flow->ssh_tls.server_info[0] != '\0') fprintf(out, "[Server: %s]", flow->ssh_tls.server_info); - if(flow->ssh_tls.server_hassh[0] != '\0') fprintf(out, "[HASSH-S: %s]", flow->ssh_tls.server_hassh); - - if(flow->ssh_tls.ja3_server[0] != '\0') fprintf(out, "[JA3S: %s%s]", flow->ssh_tls.ja3_server, - print_cipher(flow->ssh_tls.server_unsafe_cipher)); - if(flow->ssh_tls.server_organization[0] != '\0') fprintf(out, "[Organization: %s]", flow->ssh_tls.server_organization); - - if((flow->detected_protocol.master_protocol == NDPI_PROTOCOL_TLS) - || (flow->detected_protocol.app_protocol == NDPI_PROTOCOL_TLS)) { - if((flow->ssh_tls.sha1_cert_fingerprint[0] == 0) - && (flow->ssh_tls.sha1_cert_fingerprint[1] == 0) - && (flow->ssh_tls.sha1_cert_fingerprint[2] == 0)) - ; /* Looks empty */ - else { - fprintf(out, "[Certificate SHA-1: "); - for(i=0; i<20; i++) - fprintf(out, "%s%02X", (i > 0) ? ":" : "", - flow->ssh_tls.sha1_cert_fingerprint[i] & 0xFF); - fprintf(out, "]"); - } + if(flow->ssh_tls.server_info[0] != '\0') fprintf(out, "[Server: %s]", flow->ssh_tls.server_info); + if(flow->ssh_tls.server_hassh[0] != '\0') fprintf(out, "[HASSH-S: %s]", flow->ssh_tls.server_hassh); + + if(flow->ssh_tls.ja3_server[0] != '\0') fprintf(out, "[JA3S: %s%s]", flow->ssh_tls.ja3_server, + print_cipher(flow->ssh_tls.server_unsafe_cipher)); + if(flow->ssh_tls.server_organization[0] != '\0') fprintf(out, "[Organization: %s]", flow->ssh_tls.server_organization); + + if((flow->detected_protocol.master_protocol == NDPI_PROTOCOL_TLS) + || (flow->detected_protocol.app_protocol == NDPI_PROTOCOL_TLS)) { + if((flow->ssh_tls.sha1_cert_fingerprint[0] == 0) + && (flow->ssh_tls.sha1_cert_fingerprint[1] == 0) + && (flow->ssh_tls.sha1_cert_fingerprint[2] == 0)) + ; /* Looks empty */ + else { + fprintf(out, "[Certificate SHA-1: "); + for(i=0; i<20; i++) + fprintf(out, "%s%02X", (i > 0) ? ":" : "", + flow->ssh_tls.sha1_cert_fingerprint[i] & 0xFF); + fprintf(out, "]"); } + } - if(flow->ssh_tls.notBefore && flow->ssh_tls.notAfter) { - char notBefore[32], notAfter[32]; - struct tm a, b; - struct tm *before = gmtime_r(&flow->ssh_tls.notBefore, &a); - struct tm *after = gmtime_r(&flow->ssh_tls.notAfter, &b); - - strftime(notBefore, sizeof(notBefore), "%F %T", before); - strftime(notAfter, sizeof(notAfter), "%F %T", after); - - fprintf(out, "[Validity: %s - %s]", notBefore, notAfter); - } - - if(flow->ssh_tls.server_cipher != '\0') fprintf(out, "[Cipher: %s]", ndpi_cipher2str(flow->ssh_tls.server_cipher)); - if(flow->bittorent_hash[0] != '\0') fprintf(out, "[BT Hash: %s]", flow->bittorent_hash); - if(flow->dhcp_fingerprint[0] != '\0') fprintf(out, "[DHCP Fingerprint: %s]", flow->dhcp_fingerprint); - - if(flow->has_human_readeable_strings) fprintf(out, "[PLAIN TEXT (%s)]", flow->human_readeable_string_buffer); - - fprintf(out, "\n"); - } else { -#ifdef HAVE_JSON_C - jObj = json_object_new_object(); - - json_object_object_add(jObj,"protocol",json_object_new_string(ipProto2Name(flow->protocol))); - json_object_object_add(jObj,"host_a.name",json_object_new_string(flow->src_name)); - json_object_object_add(jObj,"host_a.port",json_object_new_int(ntohs(flow->src_port))); - json_object_object_add(jObj,"host_b.name",json_object_new_string(flow->dst_name)); - json_object_object_add(jObj,"host_b.port",json_object_new_int(ntohs(flow->dst_port))); - - if(flow->detected_protocol.master_protocol) - json_object_object_add(jObj,"detected.master_protocol", - json_object_new_int(flow->detected_protocol.master_protocol)); - - json_object_object_add(jObj,"detected.app_protocol", - json_object_new_int(flow->detected_protocol.app_protocol)); - - if(flow->detected_protocol.master_protocol) { - char tmp[256]; - - snprintf(tmp, sizeof(tmp), "%s.%s", - ndpi_get_proto_name(ndpi_thread_info[thread_id].workflow->ndpi_struct, - flow->detected_protocol.master_protocol), - ndpi_get_proto_name(ndpi_thread_info[thread_id].workflow->ndpi_struct, - flow->detected_protocol.app_protocol)); - - json_object_object_add(jObj,"detected.protocol.name", - json_object_new_string(tmp)); - } else - json_object_object_add(jObj,"detected.protocol.name", - json_object_new_string(ndpi_get_proto_name(ndpi_thread_info[thread_id].workflow->ndpi_struct, - flow->detected_protocol.app_protocol))); - - json_object_object_add(jObj,"packets",json_object_new_int(flow->src2dst_packets + flow->dst2src_packets)); - json_object_object_add(jObj,"bytes",json_object_new_int(flow->src2dst_bytes + flow->dst2src_bytes)); - - if(flow->host_server_name[0] != '\0') - json_object_object_add(jObj,"host.server.name",json_object_new_string(flow->host_server_name)); - - if((flow->ssh_tls.client_info[0] != '\0') || (flow->ssh_tls.server_info[0] != '\0')) { - json_object *sjObj = json_object_new_object(); + if(flow->ssh_tls.notBefore && flow->ssh_tls.notAfter) { + char notBefore[32], notAfter[32]; + struct tm a, b; + struct tm *before = gmtime_r(&flow->ssh_tls.notBefore, &a); + struct tm *after = gmtime_r(&flow->ssh_tls.notAfter, &b); - if(flow->ssh_tls.ja3_server[0] != '\0') - json_object_object_add(jObj,"ja3s",json_object_new_string(flow->ssh_tls.ja3_server)); + strftime(notBefore, sizeof(notBefore), "%F %T", before); + strftime(notAfter, sizeof(notAfter), "%F %T", after); - if(flow->ssh_tls.ja3_client[0] != '\0') - json_object_object_add(jObj,"ja3c",json_object_new_string(flow->ssh_tls.ja3_client)); - - if(flow->ssh_tls.ja3_server[0] != '\0') - json_object_object_add(jObj,"host.server.ja3",json_object_new_string(flow->ssh_tls.ja3_server)); - - if(flow->ssh_tls.client_info[0] != '\0') - json_object_object_add(sjObj, "client", json_object_new_string(flow->ssh_tls.client_info)); + fprintf(out, "[Validity: %s - %s]", notBefore, notAfter); + } - if(flow->ssh_tls.server_info[0] != '\0') - json_object_object_add(sjObj, "server", json_object_new_string(flow->ssh_tls.server_info)); + if(flow->ssh_tls.server_cipher != '\0') fprintf(out, "[Cipher: %s]", ndpi_cipher2str(flow->ssh_tls.server_cipher)); + if(flow->bittorent_hash[0] != '\0') fprintf(out, "[BT Hash: %s]", flow->bittorent_hash); + if(flow->dhcp_fingerprint[0] != '\0') fprintf(out, "[DHCP Fingerprint: %s]", flow->dhcp_fingerprint); - json_object_object_add(jObj, "ssh_tls", sjObj); - } + if(flow->has_human_readeable_strings) fprintf(out, "[PLAIN TEXT (%s)]", flow->human_readeable_string_buffer); - if(json_flag == 1) - json_object_array_add(jArray_known_flows,jObj); - else if(json_flag == 2) - json_object_array_add(jArray_unknown_flows,jObj); -#endif - } + fprintf(out, "\n"); } /* ********************************** */ @@ -1686,42 +1555,6 @@ static void updateReceivers(struct receiver **receivers, u_int32_t dst_addr, /* *********************************************** */ -#ifdef HAVE_JSON_C -static void saveReceiverStats(json_object **jObj_group, - struct receiver **receivers, - u_int64_t total_pkt_count) { - - json_object *jArray_stats = json_object_new_array(); - struct receiver *r, *tmp; - int i = 0; - - HASH_ITER(hh, *receivers, r, tmp) { - json_object *jObj_stat = json_object_new_object(); - char addr_name[48]; - - if(r->version == IPVERSION) - inet_ntop(AF_INET, &(r->addr), addr_name, sizeof(addr_name)); - else - inet_ntop(AF_INET6, &(r->addr), addr_name, sizeof(addr_name)); - - - json_object_object_add(jObj_stat,"ip.address",json_object_new_string(addr_name)); - json_object_object_add(jObj_stat,"packets.number", json_object_new_int(r->num_pkts)); - json_object_object_add(jObj_stat,"packets.percent",json_object_new_double(((double)r->num_pkts) / total_pkt_count)); - - json_object_array_add(jArray_stats, jObj_stat); - - i++; - if(i >= 10) break; - } - - json_object_object_add(*jObj_group, "top.receiver.stats", jArray_stats); -} -#endif - - -/* *********************************************** */ - static void deleteScanners(struct single_flow_info *scanners) { struct single_flow_info *s, *tmp; struct port_flow_info *p, *tmp2; @@ -1982,68 +1815,6 @@ char* formatPackets(float numPkts, char *buf) { /* *********************************************** */ /** - * @brief JSON function init - */ -#ifdef HAVE_JSON_C -static void json_init() { - jArray_known_flows = json_object_new_array(); - jArray_unknown_flows = json_object_new_array(); - jArray_topStats = json_object_new_array(); -} - -/* *********************************************** */ - -#ifdef HAVE_JSON_C -/** - * @brief JSON destroy function - */ -static void json_destroy() { - if(jArray_known_flows) { - json_object_put(jArray_known_flows); - jArray_known_flows = NULL; - } - - if(jArray_unknown_flows) { - json_object_put(jArray_unknown_flows); - jArray_unknown_flows = NULL; - } - - if(jArray_topStats) { - json_object_put(jArray_topStats); - jArray_topStats = NULL; - } -} -#endif - -/* *********************************************** */ - -static void json_open_stats_file() { - if((file_first_time && ((stats_fp = fopen(_statsFilePath,"w")) == NULL)) - || - (!file_first_time && (stats_fp = fopen(_statsFilePath,"a")) == NULL)) { - printf("Error creating/opening file %s\n", _statsFilePath); - stats_flag = 0; - } - else file_first_time = 0; -} - -/* *********************************************** */ - -static void json_close_stats_file() { - json_object *jObjFinal = json_object_new_object(); - - json_object_object_add(jObjFinal,"duration.in.seconds", - json_object_new_int(pcap_analysis_duration)); - json_object_object_add(jObjFinal,"statistics", jArray_topStats); - fprintf(stats_fp,"%s\n",json_object_to_json_string(jObjFinal)); - fclose(stats_fp); - json_object_put(jObjFinal); -} -#endif - -/* *********************************************** */ - -/** * @brief Bytes stats format */ char* formatBytes(u_int32_t howMuch, char *buf, u_int buf_len) { @@ -2082,26 +1853,6 @@ static int port_stats_sort(void *_a, void *_b) { /* *********************************************** */ -#ifdef HAVE_JSON_C -static int scanners_sort(void *_a, void *_b) { - struct single_flow_info *a = (struct single_flow_info *)_a; - struct single_flow_info *b = (struct single_flow_info *)_b; - - return(b->tot_flows - a->tot_flows); -} - -/* *********************************************** */ - -static int scanners_port_sort(void *_a, void *_b) { - struct port_flow_info *a = (struct port_flow_info *)_a; - struct port_flow_info *b = (struct port_flow_info *)_b; - - return(b->num_flows - a->num_flows); -} - -#endif -/* *********************************************** */ - static int info_pair_cmp (const void *_a, const void *_b) { struct info_pair *a = (struct info_pair *)_a; @@ -2112,171 +1863,6 @@ static int info_pair_cmp (const void *_a, const void *_b) /* *********************************************** */ -#ifdef HAVE_JSON_C -static int top_stats_sort(void *_a, void *_b) { - struct port_stats *a = (struct port_stats*)_a; - struct port_stats *b = (struct port_stats*)_b; - - return(b->num_addr - a->num_addr); -} - -/* *********************************************** */ - -/** - * @brief Get port based top statistics - */ -static int getTopStats(struct port_stats *stats) { - struct port_stats *sp, *tmp; - struct info_pair inf; - u_int64_t total_ip_addrs = 0; - - HASH_ITER(hh, stats, sp, tmp) { - qsort(sp->top_ip_addrs, MAX_NUM_IP_ADDRESS, sizeof(struct info_pair), info_pair_cmp); - inf = sp->top_ip_addrs[0]; - - if(((inf.count * 100.0)/sp->cumulative_addr) > AGGRESSIVE_PERCENT) { - sp->hasTopHost = 1; - sp->top_host = inf.addr; - sp->version = inf.version; - strncpy(sp->proto, inf.proto, sizeof(sp->proto)); - } else - sp->hasTopHost = 0; - - total_ip_addrs += sp->num_addr; - } - - return total_ip_addrs; -} - -/* *********************************************** */ - -static void saveScannerStats(json_object **jObj_group, struct single_flow_info **scanners) { - struct single_flow_info *s, *tmp; - struct port_flow_info *p, *tmp2; - char addr_name[48]; - int i = 0, j = 0; - - json_object *jArray_stats = json_object_new_array(); - - HASH_SORT(*scanners, scanners_sort); // FIX - - HASH_ITER(hh, *scanners, s, tmp) { - json_object *jObj_stat = json_object_new_object(); - json_object *jArray_ports = json_object_new_array(); - - if(s->version == IPVERSION) - inet_ntop(AF_INET, &(s->saddr), addr_name, sizeof(addr_name)); - else - inet_ntop(AF_INET6, &(s->saddr), addr_name, sizeof(addr_name)); - - json_object_object_add(jObj_stat,"ip.address",json_object_new_string(addr_name)); - json_object_object_add(jObj_stat,"total.flows.number",json_object_new_int(s->tot_flows)); - - HASH_SORT(s->ports, scanners_port_sort); - - HASH_ITER(hh, s->ports, p, tmp2) { - json_object *jObj_port = json_object_new_object(); - - json_object_object_add(jObj_port,"port",json_object_new_int(p->port)); - json_object_object_add(jObj_port,"flows.number",json_object_new_int(p->num_flows)); - - json_object_array_add(jArray_ports, jObj_port); - - j++; - if(j >= 10) break; - } - - json_object_object_add(jObj_stat,"top.dst.ports",jArray_ports); - json_object_array_add(jArray_stats, jObj_stat); - - j = 0; - i++; - if(i >= 10) break; - } - - json_object_object_add(*jObj_group, "top.scanner.stats", jArray_stats); -} - -#endif - -/* *********************************************** */ - -#ifdef HAVE_JSON_C -/* - * @brief Save Top Stats in json format - */ -static void saveTopStats(json_object **jObj_group, - struct port_stats **stats, - u_int8_t direction, - u_int64_t total_flow_count, - u_int64_t total_ip_addr) { - struct port_stats *s, *tmp; - char addr_name[48]; - int i = 0; - - json_object *jArray_stats = json_object_new_array(); - - - HASH_ITER(hh, *stats, s, tmp) { - - if((s->hasTopHost)) { - json_object *jObj_stat = json_object_new_object(); - - json_object_object_add(jObj_stat,"port",json_object_new_int(s->port)); - json_object_object_add(jObj_stat,"packets.number",json_object_new_int(s->num_pkts)); - json_object_object_add(jObj_stat,"flows.number",json_object_new_int(s->num_flows)); - json_object_object_add(jObj_stat,"flows.percent",json_object_new_double((s->num_flows*100.0)/total_flow_count)); - if(s->num_pkts) json_object_object_add(jObj_stat,"flows/packets", - json_object_new_double(((double)s->num_flows)/s->num_pkts)); - else json_object_object_add(jObj_stat,"flows.num_packets",json_object_new_double(0.0)); - - if(s->version == IPVERSION) { - inet_ntop(AF_INET, &(s->top_host), addr_name, sizeof(addr_name)); - } else { - inet_ntop(AF_INET6, &(s->top_host), addr_name, sizeof(addr_name)); - } - - json_object_object_add(jObj_stat,"aggressive.host",json_object_new_string(addr_name)); - json_object_object_add(jObj_stat,"host.app.protocol",json_object_new_string(s->proto)); - - json_object_array_add(jArray_stats, jObj_stat); - i++; - - if(i >= 10) break; - } - } - - json_object_object_add(*jObj_group, (direction == DIR_SRC) ? - "top.src.pkts.stats" : "top.dst.pkts.stats", jArray_stats); - - jArray_stats = json_object_new_array(); - i=0; - - /*sort top stats by ip addr count*/ - HASH_SORT(*stats, top_stats_sort); - - - HASH_ITER(hh, *stats, s, tmp) { - json_object *jObj_stat = json_object_new_object(); - - json_object_object_add(jObj_stat,"port",json_object_new_int(s->port)); - json_object_object_add(jObj_stat,"host.number",json_object_new_int64(s->num_addr)); - json_object_object_add(jObj_stat,"host.percent",json_object_new_double((s->num_addr*100.0)/total_ip_addr)); - json_object_object_add(jObj_stat,"flows.number",json_object_new_int(s->num_flows)); - - json_object_array_add(jArray_stats,jObj_stat); - i++; - - if(i >= 10) break; - } - - json_object_object_add(*jObj_group, (direction == DIR_SRC) ? - "top.src.host.stats" : "top.dst.host.stats", jArray_stats); -} -#endif - -/* *********************************************** */ - void printPortStats(struct port_stats *stats) { struct port_stats *s, *tmp; char addr_name[48]; @@ -2337,7 +1923,7 @@ static void printFlowsStats() { unsigned int num_ja3_client; unsigned int num_ja3_server; - if(!json_flag) fprintf(out, "\n"); + fprintf(out, "\n"); num_flows = 0; for(thread_id = 0; thread_id < num_threads; thread_id++) { @@ -2675,14 +2261,8 @@ static void printFlowsStats() { for(thread_id = 0; thread_id < num_threads; thread_id++) { if(ndpi_thread_info[thread_id].workflow->stats.protocol_counter[0 /* 0 = Unknown */] > 0) { - if(!json_flag) { - - fprintf(out, "\n\nUndetected flows:%s\n", - undetected_flows_deleted ? " (expired flows are not listed below)" : ""); - } - - if(json_flag) - json_flag = 2; + fprintf(out, "\n\nUndetected flows:%s\n", + undetected_flows_deleted ? " (expired flows are not listed below)" : ""); break; } } @@ -2730,11 +2310,6 @@ static void printResults(u_int64_t processing_time_usec, u_int64_t setup_time_us struct ndpi_stats cumulative_stats; int thread_id; char buf[32]; -#ifdef HAVE_JSON_C - FILE *json_fp = NULL; - u_int8_t dont_close_json_fp = 0; - json_object *jObj_main = NULL, *jObj_trafficStats, *jArray_detProto = NULL, *jObj; -#endif long long unsigned int breed_stats[NUM_BREEDS] = { 0 }; memset(&cumulative_stats, 0, sizeof(cumulative_stats)); @@ -2747,8 +2322,9 @@ static void printResults(u_int64_t processing_time_usec, u_int64_t setup_time_us for(i=0; i<NUM_ROOTS; i++) { ndpi_twalk(ndpi_thread_info[thread_id].workflow->ndpi_flows_root[i], node_proto_guess_walker, &thread_id); - if(verbose == 3 || stats_flag) ndpi_twalk(ndpi_thread_info[thread_id].workflow->ndpi_flows_root[i], - port_stats_walker, &thread_id); + if(verbose == 3) + ndpi_twalk(ndpi_thread_info[thread_id].workflow->ndpi_flows_root[i], + port_stats_walker, &thread_id); } /* Stats aggregation */ @@ -2789,7 +2365,6 @@ static void printResults(u_int64_t processing_time_usec, u_int64_t setup_time_us printf("\tSetup Time: %lu msec\n", (unsigned long)(setup_time_usec/1000)); printf("\tPacket Processing Time: %lu msec\n", (unsigned long)(processing_time_usec/1000)); - if(!json_flag) { printf("\nTraffic statistics:\n"); printf("\tEthernet bytes: %-13llu (includes ethernet CRC/IFC/trailer)\n", (long long unsigned int)cumulative_stats.total_wire_bytes); @@ -2841,52 +2416,11 @@ static void printResults(u_int64_t processing_time_usec, u_int64_t setup_time_us } if(enable_protocol_guess) - printf("\tGuessed flow protos: %-13u\n", cumulative_stats.guessed_flow_protocols); - } + printf("\tGuessed flow protos: %-13u\n", cumulative_stats.guessed_flow_protocols); } - if(json_flag) { -#ifdef HAVE_JSON_C - if(!strcmp(_jsonFilePath, "-")) - json_fp = stderr, dont_close_json_fp = 1; - else if((json_fp = fopen(_jsonFilePath,"w")) == NULL) { - printf("Error creating .json file %s\n", _jsonFilePath); - json_flag = 0; - } - if(json_flag) { - jObj_main = json_object_new_object(); - jObj_trafficStats = json_object_new_object(); - jArray_detProto = json_object_new_array(); - - json_object_object_add(jObj_trafficStats,"ethernet.bytes",json_object_new_int64(cumulative_stats.total_wire_bytes)); - json_object_object_add(jObj_trafficStats,"discarded.bytes",json_object_new_int64(cumulative_stats.total_discarded_bytes)); - json_object_object_add(jObj_trafficStats,"ip.packets",json_object_new_int64(cumulative_stats.ip_packet_count)); - json_object_object_add(jObj_trafficStats,"total.packets",json_object_new_int64(cumulative_stats.raw_packet_count)); - json_object_object_add(jObj_trafficStats,"ip.bytes",json_object_new_int64(cumulative_stats.total_ip_bytes)); - json_object_object_add(jObj_trafficStats,"avg.pkt.size",json_object_new_int(cumulative_stats.total_ip_bytes/cumulative_stats.raw_packet_count)); - json_object_object_add(jObj_trafficStats,"unique.flows",json_object_new_int(cumulative_stats.ndpi_flow_count)); - json_object_object_add(jObj_trafficStats,"tcp.pkts",json_object_new_int64(cumulative_stats.tcp_count)); - json_object_object_add(jObj_trafficStats,"udp.pkts",json_object_new_int64(cumulative_stats.udp_count)); - json_object_object_add(jObj_trafficStats,"vlan.pkts",json_object_new_int64(cumulative_stats.vlan_count)); - json_object_object_add(jObj_trafficStats,"mpls.pkts",json_object_new_int64(cumulative_stats.mpls_count)); - json_object_object_add(jObj_trafficStats,"pppoe.pkts",json_object_new_int64(cumulative_stats.pppoe_count)); - json_object_object_add(jObj_trafficStats,"fragmented.pkts",json_object_new_int64(cumulative_stats.fragmented_count)); - json_object_object_add(jObj_trafficStats,"max.pkt.size",json_object_new_int(cumulative_stats.max_packet_len)); - json_object_object_add(jObj_trafficStats,"pkt.len_min64",json_object_new_int64(cumulative_stats.packet_len[0])); - json_object_object_add(jObj_trafficStats,"pkt.len_64_128",json_object_new_int64(cumulative_stats.packet_len[1])); - json_object_object_add(jObj_trafficStats,"pkt.len_128_256",json_object_new_int64(cumulative_stats.packet_len[2])); - json_object_object_add(jObj_trafficStats,"pkt.len_256_1024",json_object_new_int64(cumulative_stats.packet_len[3])); - json_object_object_add(jObj_trafficStats,"pkt.len_1024_1500",json_object_new_int64(cumulative_stats.packet_len[4])); - json_object_object_add(jObj_trafficStats,"pkt.len_grt1500",json_object_new_int64(cumulative_stats.packet_len[5])); - json_object_object_add(jObj_trafficStats,"guessed.flow.protos",json_object_new_int(cumulative_stats.guessed_flow_protocols)); - - json_object_object_add(jObj_main,"traffic.statistics",jObj_trafficStats); - } -#endif - } - - if((!json_flag) && (!quiet_mode)) printf("\n\nDetected protocols:\n"); + if(!quiet_mode) printf("\n\nDetected protocols:\n"); for(i = 0; i <= ndpi_get_num_supported_protocols(ndpi_thread_info[0].workflow->ndpi_struct); i++) { ndpi_protocol_breed_t breed = ndpi_get_proto_breed(ndpi_thread_info[0].workflow->ndpi_struct, i); @@ -2900,34 +2434,20 @@ static void printResults(u_int64_t processing_time_usec, u_int64_t setup_time_us (long long unsigned int)cumulative_stats.protocol_counter_bytes[i], cumulative_stats.protocol_flows[i]); - if((!json_flag) && (!quiet_mode)) { + if((!quiet_mode)) { printf("\t%-20s packets: %-13llu bytes: %-13llu " "flows: %-13u\n", ndpi_get_proto_name(ndpi_thread_info[0].workflow->ndpi_struct, i), (long long unsigned int)cumulative_stats.protocol_counter[i], (long long unsigned int)cumulative_stats.protocol_counter_bytes[i], cumulative_stats.protocol_flows[i]); - } else { -#ifdef HAVE_JSON_C - if(json_fp) { - jObj = json_object_new_object(); - - json_object_object_add(jObj,"name",json_object_new_string(ndpi_get_proto_name(ndpi_thread_info[0].workflow->ndpi_struct, i))); - json_object_object_add(jObj,"breed",json_object_new_string(ndpi_get_proto_breed_name(ndpi_thread_info[0].workflow->ndpi_struct, breed))); - json_object_object_add(jObj,"packets",json_object_new_int64(cumulative_stats.protocol_counter[i])); - json_object_object_add(jObj,"bytes",json_object_new_int64(cumulative_stats.protocol_counter_bytes[i])); - json_object_object_add(jObj,"flows",json_object_new_int(cumulative_stats.protocol_flows[i])); - - json_object_array_add(jArray_detProto,jObj); - } -#endif } total_flow_bytes += cumulative_stats.protocol_counter_bytes[i]; } } - if((!json_flag) && (!quiet_mode)) { + if((!quiet_mode)) { printf("\n\nProtocol statistics:\n"); for(i=0; i < NUM_BREEDS; i++) { @@ -2943,25 +2463,10 @@ static void printResults(u_int64_t processing_time_usec, u_int64_t setup_time_us printFlowsStats(); - if(json_flag != 0) { -#ifdef HAVE_JSON_C - json_object_object_add(jObj_main,"detected.protos",jArray_detProto); - json_object_object_add(jObj_main,"known.flows",jArray_known_flows); - - if(json_object_array_length(jArray_unknown_flows) != 0) - json_object_object_add(jObj_main,"unknown.flows",jArray_unknown_flows); - - fprintf(json_fp,"%s\n",json_object_to_json_string(jObj_main)); - if(!dont_close_json_fp) fclose(json_fp); -#endif - } - - if(stats_flag || verbose == 3) { + if(verbose == 3) { HASH_SORT(srcStats, port_stats_sort); HASH_SORT(dstStats, port_stats_sort); - } - if(verbose == 3) { printf("\n\nSource Ports Stats:\n"); printPortStats(srcStats); @@ -2969,39 +2474,6 @@ static void printResults(u_int64_t processing_time_usec, u_int64_t setup_time_us printPortStats(dstStats); } - if(stats_flag) { -#ifdef HAVE_JSON_C - json_object *jObj_stats = json_object_new_object(); - char timestamp[64]; - int count; - - strftime(timestamp, sizeof(timestamp), "%FT%TZ", localtime(&pcap_start.tv_sec)); - json_object_object_add(jObj_stats, "time", json_object_new_string(timestamp)); - - saveScannerStats(&jObj_stats, &scannerHosts); - - if((count = HASH_COUNT(topReceivers)) == 0) { - HASH_SORT(receivers, receivers_sort); - saveReceiverStats(&jObj_stats, &receivers, cumulative_stats.ip_packet_count); - } - else{ - HASH_SORT(topReceivers, receivers_sort); - saveReceiverStats(&jObj_stats, &topReceivers, cumulative_stats.ip_packet_count); - } - - u_int64_t total_src_addr = getTopStats(srcStats); - u_int64_t total_dst_addr = getTopStats(dstStats); - - saveTopStats(&jObj_stats, &srcStats, DIR_SRC, - cumulative_stats.ndpi_flow_count, total_src_addr); - - saveTopStats(&jObj_stats, &dstStats, DIR_DST, - cumulative_stats.ndpi_flow_count, total_dst_addr); - - json_object_array_add(jArray_topStats, jObj_stats); -#endif - } - free_stats: if(scannerHosts) { deleteScanners(scannerHosts); @@ -3142,17 +2614,17 @@ static pcap_t * openPcapFileOrDevice(u_int16_t thread_id, const u_char * pcap_fi printf("ERROR: could not open playlist %s: %s\n", filename, pcap_error_buffer); exit(-1); } else { - if((!json_flag) && (!quiet_mode)) + if((!quiet_mode)) printf("Reading packets from playlist %s...\n", pcap_file); } } else { - if((!json_flag) && (!quiet_mode)) + if((!quiet_mode)) printf("Reading packets from pcap file %s...\n", pcap_file); } } else { live_capture = 1; - if((!json_flag) && (!quiet_mode)) { + if((!quiet_mode)) { #ifdef USE_DPDK printf("Capturing from DPDK (port 0)...\n"); #else @@ -3165,7 +2637,7 @@ static pcap_t * openPcapFileOrDevice(u_int16_t thread_id, const u_char * pcap_fi #endif /* !DPDK */ if(capture_for > 0) { - if((!json_flag) && (!quiet_mode)) + if((!quiet_mode)) printf("Capturing traffic up to %u seconds\n", (unsigned int)capture_for); #ifndef WIN32 @@ -3321,11 +2793,11 @@ void * processing_thread(void *_thread_id) { if(pthread_setaffinity_np(pthread_self(), sizeof(cpu_set_t), &cpuset) != 0) fprintf(stderr, "Error while binding thread %ld to core %d\n", thread_id, core_affinity[thread_id]); else { - if((!json_flag) && (!quiet_mode)) printf("Running thread %ld on core %d...\n", thread_id, core_affinity[thread_id]); + if((!quiet_mode)) printf("Running thread %ld on core %d...\n", thread_id, core_affinity[thread_id]); } } else #endif - if((!json_flag) && (!quiet_mode)) printf("Running thread %ld...\n", thread_id); + if((!quiet_mode)) printf("Running thread %ld...\n", thread_id); #ifdef USE_DPDK while(dpdk_run_capture) { @@ -3380,11 +2852,6 @@ void test_lib() { u_int64_t processing_time_usec, setup_time_usec; long thread_id; -#ifdef HAVE_JSON_C - json_init(); - if(stats_flag) json_open_stats_file(); -#endif - #ifdef DEBUG_TRACE if(trace) fprintf(trace, "Num threads: %d\n", num_threads); #endif @@ -3435,22 +2902,12 @@ void test_lib() { /* Printing cumulative results */ printResults(processing_time_usec, setup_time_usec); - if(stats_flag) { -#ifdef HAVE_JSON_C - json_close_stats_file(); -#endif - } - for(thread_id = 0; thread_id < num_threads; thread_id++) { if(ndpi_thread_info[thread_id].workflow->pcap_handle != NULL) pcap_close(ndpi_thread_info[thread_id].workflow->pcap_handle); terminateDetection(thread_id); } - -#ifdef HAVE_JSON_C - json_destroy(); -#endif } /* *********************************************** */ @@ -3589,135 +3046,6 @@ void analyzeUnitTest() { } /* *********************************************** */ - -/** - * @brief Produce bpf filter to filter ports and hosts - * in order to remove a peak in terms of number of packets - * sent by source hosts. - */ -#ifdef HAVE_JSON_C -void bpf_filter_pkt_peak_filter(json_object **jObj_bpfFilter, - int port_array[], int p_size, - const char *src_host_array[16], - int sh_size, - const char *dst_host_array[16], - int dh_size) { - char filter[2048] = { '\0' }; - int produced = 0; - int i = 0, l = 0; - - if(port_array[0] != INIT_VAL) { - strcpy(filter, "not (src port "); - - l = strlen(filter); - - while(i < p_size && port_array[i] != INIT_VAL) { - if(i+1 == p_size || port_array[i+1] == INIT_VAL) - snprintf(&filter[l], sizeof(filter)-l, "%d", port_array[i]); - else - snprintf(&filter[l], sizeof(filter)-l, "%d or ", port_array[i]); - - i++; - } - - l += snprintf(&filter[l], sizeof(filter)-l, "%s", ")"); - produced = 1; - } - - - if(src_host_array[0] != NULL) { - if(port_array[0] != INIT_VAL) - l += snprintf(&filter[l], sizeof(filter)-l, " and not (src "); - else - l += snprintf(&filter[l], sizeof(filter)-l, "not (src "); - - i = 0; - - while(i < sh_size && src_host_array[i] != NULL) { - if(i+1 == sh_size || src_host_array[i+1] == NULL) - l += snprintf(&filter[l], sizeof(filter)-l, "%s", src_host_array[i]); - else - l += snprintf(&filter[l], sizeof(filter)-l, "%s or ", src_host_array[i]); - - i++; - } - - l += snprintf(&filter[l], sizeof(filter)-l, "%s", ")"); - produced = 1; - } - - if(dst_host_array[0] != NULL) { - if(port_array[0] != INIT_VAL || src_host_array[0] != NULL) - l += snprintf(&filter[l], sizeof(filter)-l, " and not (dst "); - else - l += snprintf(&filter[l], sizeof(filter)-l, "not (dst "); - - i=0; - - while(i < dh_size && dst_host_array[i] != NULL) { - if(i+1 == dh_size || dst_host_array[i+1] == NULL) - l += snprintf(&filter[l], sizeof(filter)-l, "%s", dst_host_array[i]); - else - l += snprintf(&filter[l], sizeof(filter)-l, "%s or ", dst_host_array[i]); - - i++; - } - - l += snprintf(&filter[l], sizeof(filter)-l, "%s", ")"); - produced = 1; - } - - if(produced) - json_object_object_add(*jObj_bpfFilter, "pkt.peak.filter", json_object_new_string(filter)); - else - json_object_object_add(*jObj_bpfFilter, "pkt.peak.filter", json_object_new_string("")); -} -#endif - -/* *********************************************** */ -/** - * @brief Produce bpf filter to filter ports and hosts - * in order to remove a peak in terms of number of source - * addresses. - */ -#ifdef HAVE_JSON_C -void bpf_filter_host_peak_filter(json_object **jObj_bpfFilter, - const char *host_array[16], - int h_size) { - char filter[2048]; - int produced = 0; - int i = 0; - - - if(host_array[0] != NULL) { - int l; - - strcpy(filter, "not (dst "); - - while(i < h_size && host_array[i] != NULL) { - l = strlen(filter); - - if(i+1 == h_size || host_array[i+1] == NULL) - snprintf(&filter[l], sizeof(filter)-l, "%s", host_array[i]); - else - snprintf(&filter[l], sizeof(filter)-l, "%s or ", host_array[i]); - - i++; - } - - l = strlen(filter); - snprintf(&filter[l], sizeof(filter)-l, "%s", ")"); - produced = 1; - } - - if(produced) - json_object_object_add(*jObj_bpfFilter, "host.peak.filter", json_object_new_string(filter)); - else - json_object_object_add(*jObj_bpfFilter, "host.peak.filter", json_object_new_string("")); -} -#endif - -/* *********************************************** */ /** * @brief Initialize port array */ @@ -3781,370 +3109,6 @@ void bpf_filter_port_array_add(int filter_array[], int size, int port) { exit(-1); } - -/* *********************************************** */ -#ifdef HAVE_JSON_C -/* - * @brief returns average value for a given field - */ -float getAverage(struct json_object *jObj_stat, char *field) { - json_object *field_stat; - json_bool res; - float sum = 0; - int r; - int j = 0; - - if((r = strcmp(field, "top.scanner.stats")) == 0) { - for(j=0; j<json_object_array_length(jObj_stat); j++) { - field_stat = json_object_array_get_idx(jObj_stat, j); - json_object *jObj_tot_flows_number; - - if((res = json_object_object_get_ex(field_stat, "total.flows.number", &jObj_tot_flows_number)) == 0) { - fprintf(stderr, "ERROR: can't get \"total.flows.number\", use -x flag only with .json files generated by ndpiReader -b flag.\n"); - exit(-1); - } - u_int32_t tot_flows_number = json_object_get_int(jObj_tot_flows_number); - - sum += tot_flows_number; - } - } else if((r = strcmp(field, "top.src.pkts.stats")) == 0) { - for(j=0; j<json_object_array_length(jObj_stat); j++) { - field_stat = json_object_array_get_idx(jObj_stat, j); - json_object *jObj_packets_number; - - if((res = json_object_object_get_ex(field_stat, "packets.number", &jObj_packets_number)) == 0) { - fprintf(stderr, "ERROR: can't get \"packets.number\", use -x flag only with .json files generated by ndpiReader -b flag.\n"); - exit(-1); - } - u_int32_t packets_number = json_object_get_int(jObj_packets_number); - - sum += packets_number; - } - } - - if(j == 0) return 0.0; - - return sum/j; -} -#endif -/* *********************************************** */ -#ifdef HAVE_JSON_C -/* - * @brief returns standard deviation for a given - * field and it's average value. - */ -float getStdDeviation(struct json_object *jObj_stat, float average, char *field) { - json_object *field_stat; - json_bool res; - float sum = 0; - int j = 0; - int r; - - if((r = strcmp(field, "top.scanner.stats")) == 0) { - for(; j<json_object_array_length(jObj_stat); j++) { - field_stat = json_object_array_get_idx(jObj_stat, j); - json_object *jObj_tot_flows_number; - - if((res = json_object_object_get_ex(field_stat, "total.flows.number", &jObj_tot_flows_number)) == 0) { - fprintf(stderr, "ERROR: can't get \"total.flows.number\", use -x flag only with .json files generated by ndpiReader -b flag.\n"); - exit(-1); - } - u_int32_t tot_flows_number = json_object_get_int(jObj_tot_flows_number); - - sum += pow((tot_flows_number - average), 2); - } - } - - return sqrt(sum/(float)j); -} - -#endif - -/* *********************************************** */ - -#ifdef HAVE_JSON_C -void getSourcePorts(struct json_object *jObj_stat, int srcPortArray[], int size, float threshold) { - int j; - - for(j=0; j<json_object_array_length(jObj_stat); j++) { - json_object *src_pkts_stat = json_object_array_get_idx(jObj_stat, j); - json_object *jObj_packets_number; - json_object *jObj_flows_percent; - json_object *jObj_flows_packets; - json_object *jObj_port; - json_bool res; - - if((res = json_object_object_get_ex(src_pkts_stat, "packets.number", &jObj_packets_number)) == 0) { - fprintf(stderr, "ERROR: can't get \"packets.number\", use -x flag only with .json files generated by ndpiReader -b flag.\n"); - exit(-1); - } - u_int32_t packets_number = json_object_get_int(jObj_packets_number); - - if((res = json_object_object_get_ex(src_pkts_stat, "flows.percent", &jObj_flows_percent)) == 0) { - fprintf(stderr, "ERROR: can't get \"flows.percent\", use -x flag only with .json files generated by ndpiReader -b flag.\n"); - exit(-1); - } - double flows_percent = json_object_get_double(jObj_flows_percent); - - - if((res = json_object_object_get_ex(src_pkts_stat, "flows/packets", &jObj_flows_packets)) == 0) { - fprintf(stderr, "ERROR: can't get \"flows/packets\", use -x flag only with .json files generated by ndpiReader -b flag.\n"); - exit(-1); - } - double flows_packets = json_object_get_double(jObj_flows_packets); - - - if((flows_packets > FLOWS_PACKETS_THRESHOLD) - && (flows_percent >= FLOWS_PERCENT_THRESHOLD) - && packets_number >= threshold) { - if((res = json_object_object_get_ex(src_pkts_stat, "port", &jObj_port)) == 0) { - fprintf(stderr, "ERROR: can't get \"port\", use -x flag only with .json files generated by ndpiReader -b flag.\n"); - exit(-1); - } - int port = json_object_get_int(jObj_port); - - bpf_filter_port_array_add(srcPortArray, size, port); - } - } -} -#endif - -/* *********************************************** */ - -#ifdef HAVE_JSON_C -void getReceiverHosts(struct json_object *jObj_stat, const char *dstHostArray[16], int size) { - int j; - - for(j=0; j<json_object_array_length(jObj_stat); j++) { - json_object *scanner_stat = json_object_array_get_idx(jObj_stat, j); - json_object *jObj_host_address; - json_object *jObj_pkts_percent; - json_bool res; - - if((res = json_object_object_get_ex(scanner_stat, "packets.percent", &jObj_pkts_percent)) == 0) { - fprintf(stderr, "ERROR: can't get \"packets.percent\", use -x flag only with .json files generated by ndpiReader -b flag.\n"); - exit(-1); - } - double pkts_percent = json_object_get_double(jObj_pkts_percent); - - - if(pkts_percent > PKTS_PERCENT_THRESHOLD) { - if((res = json_object_object_get_ex(scanner_stat, "ip.address", &jObj_host_address)) == 0) { - fprintf(stderr, "ERROR: can't get \"ip.address, use -x flag only with .json files generated by ndpiReader -b flag.\n"); - exit(-1); - } - const char *host_address = json_object_get_string(jObj_host_address); - - bpf_filter_host_array_add(dstHostArray, size, host_address); - } - } -} -#endif - -/* *********************************************** */ - -#ifdef HAVE_JSON_C -void getScannerHosts(struct json_object *jObj_stat, int duration, - const char *srcHostArray[48], int size, - float threshold) { - int j; - - for(j=0; j<json_object_array_length(jObj_stat); j++) { - json_object *scanner_stat = json_object_array_get_idx(jObj_stat, j); - json_object *jObj_host_address; - json_object *jObj_tot_flows_number; - json_bool res; - - - if((res = json_object_object_get_ex(scanner_stat, "total.flows.number", &jObj_tot_flows_number)) == 0) { - fprintf(stderr, "ERROR: can't get \"total.flows.number\", use -x flag only with .json files generated by ndpiReader -b flag.\n"); - exit(-1); - } - u_int32_t tot_flows_number = json_object_get_int(jObj_tot_flows_number); - - - if(((tot_flows_number/(float)duration) > FLOWS_THRESHOLD) && tot_flows_number > threshold) { - if((res = json_object_object_get_ex(scanner_stat, "ip.address", &jObj_host_address)) == 0) { - fprintf(stderr, "ERROR: can't get \"ip.address\", use -x flag only with .json files generated by ndpiReader -b flag.\n"); - exit(-1); - } - const char *host_address = json_object_get_string(jObj_host_address); - - bpf_filter_host_array_add(srcHostArray, size, host_address); - - } - } -} -#endif - -/* *********************************************** */ - -#ifdef HAVE_JSON_C -void getDestinationHosts(struct json_object *jObj_stat, int duration, - const char *dstHostArray[16], int size) { - int j; - - for(j=0; j<json_object_array_length(jObj_stat); j++) { - json_object *scanner_stat = json_object_array_get_idx(jObj_stat, j); - json_object *jObj_host_address; - json_object *jObj_flows_percent; - json_bool res; - - - if((res = json_object_object_get_ex(scanner_stat, "flows.percent", &jObj_flows_percent)) == 0) { - fprintf(stderr, "ERROR: can't get \"flows.percent\", use -x flag only with .json files generated by ndpiReader -b flag.\n"); - exit(-1); - } - double flows_percent = json_object_get_double(jObj_flows_percent); - - - if(flows_percent > FLOWS_PERCENT_THRESHOLD_2) { - if((res = json_object_object_get_ex(scanner_stat, "aggressive.host", &jObj_host_address)) == 0) { - fprintf(stderr, "ERROR: can't get \"aggressive.host\", use -x flag only with .json files generated by ndpiReader -b flag.\n"); - exit(-1); - } - const char *host_address = json_object_get_string(jObj_host_address); - - bpf_filter_host_array_add(dstHostArray, size, host_address); - - } - } -} -#endif - -/* *********************************************** */ - -#ifdef HAVE_JSON_C -static void produceBpfFilter(char *filePath) { - json_object *jObj; /* entire json object from file */ - json_object *jObj_duration; - json_object *jObj_statistics; /* json array */ - json_bool res; - int filterSrcPorts[PORT_ARRAY_SIZE]; - const char *filterSrcHosts[48]; - const char *filterDstHosts[48]; - const char *filterPktDstHosts[48]; - struct stat statbuf; - FILE *fp = NULL; - char _filterFilePath[1024]; - json_object *jObj_bpfFilter; - void *fmap; - int fsock; - float average; - float deviation; - int duration; - int typeCheck; - int array_len; - int i; - - if((fsock = open(filePath, O_RDONLY)) == -1) { - fprintf(stderr,"error opening file %s\n", filePath); - exit(-1); - } - - if(fstat(fsock, &statbuf) == -1) { - fprintf(stderr,"error getting file stat\n"); - exit(-1); - } - - if((fmap = mmap(NULL, statbuf.st_size, PROT_READ, MAP_PRIVATE, fsock, 0)) == MAP_FAILED) { - fprintf(stderr,"error mmap is failed\n"); - exit(-1); - } - - if((jObj = json_tokener_parse(fmap)) == NULL) { - fprintf(stderr,"ERROR: invalid json file. Use -x flag only with .json files generated by ndpiReader -b flag.\n"); - exit(-1); - } - - - if((res = json_object_object_get_ex(jObj, "duration.in.seconds", &jObj_duration)) == 0) { - fprintf(stderr,"ERROR: can't get \"duration.in.seconds\", use -x flag only with .json files generated by ndpiReader -b flag.\n"); - exit(-1); - } - duration = json_object_get_int(jObj_duration); - - - if((res = json_object_object_get_ex(jObj, "statistics", &jObj_statistics)) == 0) { - fprintf(stderr,"ERROR: can't get \"statistics\", use -x flag only with .json files generated by ndpiReader -b flag.\n"); - exit(-1); - } - - if((typeCheck = json_object_is_type(jObj_statistics, json_type_array)) == 0) { - fprintf(stderr,"ERROR: invalid json file. Use -x flag only with .json files generated by ndpiReader -b flag.\n"); - exit(-1); - } - array_len = json_object_array_length(jObj_statistics); - - - bpf_filter_port_array_init(filterSrcPorts, PORT_ARRAY_SIZE); - bpf_filter_host_array_init(filterSrcHosts, HOST_ARRAY_SIZE); - bpf_filter_host_array_init(filterDstHosts, HOST_ARRAY_SIZE); - bpf_filter_host_array_init(filterPktDstHosts, HOST_ARRAY_SIZE/2); - - for(i=0; i<array_len; i++) { - json_object *stats = json_object_array_get_idx(jObj_statistics, i); - json_object *val; - - if((res = json_object_object_get_ex(stats, "top.scanner.stats", &val)) == 0) { - fprintf(stderr,"ERROR: can't get \"top.scanner.stats\", use -x flag only with .json files generated by ndpiReader -b flag.\n"); - exit(-1); - } - - if((average = getAverage(val, "top.scanner.stats")) != 0) { - deviation = getStdDeviation(val, average, "top.scanner.stats"); - getScannerHosts(val, duration, filterSrcHosts, HOST_ARRAY_SIZE, average+deviation); - } - - - if((res = json_object_object_get_ex(stats, "top.receiver.stats", &val)) == 0) { - fprintf(stderr,"ERROR: can't get \"top.receiver.stats\", use -x flag only with .json files generated by ndpiReader -b flag.\n"); - exit(-1); - } - getReceiverHosts(val, filterPktDstHosts, HOST_ARRAY_SIZE/2); - - - if((res = json_object_object_get_ex(stats, "top.src.pkts.stats", &val)) == 0) { - fprintf(stderr,"ERROR: can't get \"top.src.pkts.stats\", use -x flag only with .json files generated by ndpiReader -b flag.\n"); - exit(-1); - } - - if((average = getAverage(val, "top.src.pkts.stats")) != 0) - getSourcePorts(val, filterSrcPorts, PORT_ARRAY_SIZE, average); - - - if((res = json_object_object_get_ex(stats, "top.dst.pkts.stats", &val)) == 0) { - fprintf(stderr,"ERROR: can't get \"top.dst.pkts.stats\", use -x flag only with .json files generated by ndpiReader -b flag.\n"); - exit(-1); - } - getDestinationHosts(val, duration, filterDstHosts, HOST_ARRAY_SIZE); - } - - - snprintf(_filterFilePath, sizeof(_filterFilePath), "%s.bpf", filePath); - - if((fp = fopen(_filterFilePath,"w")) == NULL) { - printf("Error creating .json file %s\n", _filterFilePath); - exit(-1); - } - - jObj_bpfFilter = json_object_new_object(); - - bpf_filter_pkt_peak_filter(&jObj_bpfFilter, filterSrcPorts, PORT_ARRAY_SIZE, - filterSrcHosts, HOST_ARRAY_SIZE, filterPktDstHosts, HOST_ARRAY_SIZE/2); - - bpf_filter_host_peak_filter(&jObj_bpfFilter, filterDstHosts, HOST_ARRAY_SIZE); - - fprintf(fp,"%s\n",json_object_to_json_string(jObj_bpfFilter)); - fclose(fp); - - printf("created: %s\n", _filterFilePath); - - json_object_put(jObj); /* free memory */ -} -#endif - - /* *********************************************** */ @@ -4177,14 +3141,7 @@ int orginal_main(int argc, char **argv) { parseOptions(argc, argv); - if(bpf_filter_flag) { -#ifdef HAVE_JSON_C - produceBpfFilter(_diagnoseFilePath); - return 0; -#endif - } - - if((!json_flag) && (!quiet_mode)) { + if(!quiet_mode) { printf("\n-----------------------------------------------------------\n" "* NOTE: This is demo app to show *some* nDPI features.\n" "* In this demo we have implemented only some basic features\n" |