From d8b2189cc30f675fba46b072d162dc5943b1c362 Mon Sep 17 00:00:00 2001 From: Luca Deri Date: Fri, 14 Apr 2017 00:07:46 +0200 Subject: Initial wireshark integration via extcap interface --- example/ndpi_util.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'example/ndpi_util.h') diff --git a/example/ndpi_util.h b/example/ndpi_util.h index 1c092cbfa..4895cdcb1 100644 --- a/example/ndpi_util.h +++ b/example/ndpi_util.h @@ -138,9 +138,9 @@ void ndpi_free_flow_info_half(struct ndpi_flow_info *flow); /* Process a packet and update the workflow */ -void ndpi_workflow_process_packet (struct ndpi_workflow * workflow, - const struct pcap_pkthdr *header, - const u_char *packet); +struct ndpi_proto ndpi_workflow_process_packet(struct ndpi_workflow * workflow, + const struct pcap_pkthdr *header, + const u_char *packet); /* flow callbacks for complete detected flow -- cgit v1.2.3 From 4c5de9ef8e9e14289ce9024349390fe06b59c769 Mon Sep 17 00:00:00 2001 From: Luca Date: Mon, 17 Apr 2017 09:38:30 +0200 Subject: Added ethernet checksum reforging and nDPI protocol export in nDPI-extcap reader --- example/ndpiReader.c | 26 +++++++++++++++++++++----- example/ndpi_util.c | 40 ++++++++++++++++++++++++++++++++++++++++ example/ndpi_util.h | 2 +- 3 files changed, 62 insertions(+), 6 deletions(-) (limited to 'example/ndpi_util.h') diff --git a/example/ndpiReader.c b/example/ndpiReader.c index b32650871..9eab4500f 100644 --- a/example/ndpiReader.c +++ b/example/ndpiReader.c @@ -81,7 +81,13 @@ static time_t capture_for = 0; static time_t capture_until = 0; static u_int32_t num_flows; +struct ndpi_packet_trailer { + u_int32_t magic; /* 0x19682017 */ + u_int16_t master_protocol /* e.g. HTTP */, app_protocol /* e.g. FaceBook */; +}; + static pcap_dumper_t *extcap_dumper = NULL; +static char extcap_buf[2048]; static char *extcap_capture_fifo = NULL; static u_int16_t extcap_packet_filter = (u_int16_t)-1; @@ -422,7 +428,7 @@ static void parseOptions(int argc, char **argv) { case '9': extcap_packet_filter = atoi(optarg); break; - + default: help(0); break; @@ -1383,12 +1389,22 @@ static void pcap_packet_callback_checked(u_char *args, ) ) { struct pcap_pkthdr *h = (struct pcap_pkthdr*)header; - + uint32_t *crc, delta = sizeof(struct ndpi_packet_trailer) + 4 /* ethernet trailer */; + struct ndpi_packet_trailer *trailer = (struct ndpi_packet_trailer*)&extcap_buf[h->caplen]; + + memcpy(extcap_buf, packet, h->caplen); + trailer->magic = 0x19682017; + trailer->master_protocol = p.master_protocol, trailer->app_protocol = p.app_protocol; + crc = (uint32_t*)&extcap_buf[h->caplen+sizeof(struct ndpi_packet_trailer)]; + *crc = 0; + ethernet_crc32((const void*)extcap_buf, h->caplen+sizeof(struct ndpi_packet_trailer), crc); + h->caplen += delta, h->len += delta; + #ifdef DEBUG_TRACE if(trace) fprintf(trace, "Dumping %u bytes packet\n", header->caplen); #endif - // h->caplen += 8, h->len += 8; - pcap_dump((u_char*)extcap_dumper, h, packet); + + pcap_dump((u_char*)extcap_dumper, h, (const u_char *)extcap_buf); } /* check for buffer changes */ @@ -1503,7 +1519,7 @@ void test_lib() { exit(-1); } } - + gettimeofday(&end, NULL); tot_usec = end.tv_sec*1000000 + end.tv_usec - (begin.tv_sec*1000000 + begin.tv_usec); diff --git a/example/ndpi_util.c b/example/ndpi_util.c index 8fe80111a..cd3c8a1e3 100644 --- a/example/ndpi_util.c +++ b/example/ndpi_util.c @@ -909,3 +909,43 @@ struct ndpi_proto ndpi_workflow_process_packet (struct ndpi_workflow * workflow, return(packet_processing(workflow, time, vlan_id, iph, iph6, ip_offset, header->len - ip_offset, header->len)); } + +/* ********************************************************** */ +/* http://home.thep.lu.se/~bjorn/crc/crc32_fast.c */ +/* ********************************************************** */ + +static uint32_t crc32_for_byte(uint32_t r) { + for(int j = 0; j < 8; ++j) + r = (r & 1? 0: (uint32_t)0xEDB88320L) ^ r >> 1; + return r ^ (uint32_t)0xFF000000L; +} + +/* Any unsigned integer type with at least 32 bits may be used as + * accumulator type for fast crc32-calulation, but unsigned long is + * probably the optimal choice for most systems. */ +typedef unsigned long accum_t; + +static void init_tables(uint32_t* table, uint32_t* wtable) { + for(size_t i = 0; i < 0x100; ++i) + table[i] = crc32_for_byte(i); + for(size_t k = 0; k < sizeof(accum_t); ++k) + for(size_t w, i = 0; i < 0x100; ++i) { + for(size_t j = w = 0; j < sizeof(accum_t); ++j) + w = table[(uint8_t)(j == k? w ^ i: w)] ^ w >> 8; + wtable[(k << 8) + i] = w ^ (k? wtable[0]: 0); + } +} + +void ethernet_crc32(const void* data, size_t n_bytes, uint32_t* crc) { + static uint32_t table[0x100], wtable[0x100*sizeof(accum_t)]; + size_t n_accum = n_bytes/sizeof(accum_t); + if(!*table) + init_tables(table, wtable); + for(size_t i = 0; i < n_accum; ++i) { + accum_t a = *crc ^ ((accum_t*)data)[i]; + for(size_t j = *crc = 0; j < sizeof(accum_t); ++j) + *crc ^= wtable[(j << 8) + (uint8_t)(a >> 8*j)]; + } + for(size_t i = n_accum*sizeof(accum_t); i < n_bytes; ++i) + *crc = table[(uint8_t)*crc ^ ((uint8_t*)data)[i]] ^ *crc >> 8; +} diff --git a/example/ndpi_util.h b/example/ndpi_util.h index 4895cdcb1..a8e21d673 100644 --- a/example/ndpi_util.h +++ b/example/ndpi_util.h @@ -160,5 +160,5 @@ static inline void ndpi_workflow_set_flow_giveup_callback(struct ndpi_workflow * /* compare two nodes in workflow */ int ndpi_workflow_node_cmp(const void *a, const void *b); void process_ndpi_collected_info(struct ndpi_workflow * workflow, struct ndpi_flow_info *flow); - +void ethernet_crc32(const void* data, size_t n_bytes, uint32_t* crc); #endif -- cgit v1.2.3 From a9c01ded174ed380a2d135cfb9b903f616b0e175 Mon Sep 17 00:00:00 2001 From: Luca Deri Date: Wed, 19 Apr 2017 21:55:49 +0200 Subject: ndpiReader now prints (-v) the flows with the correct direction --- example/ndpiReader.c | 28 +++++++++++++++++----------- example/ndpi_util.c | 22 ++++++++++------------ example/ndpi_util.h | 2 +- 3 files changed, 28 insertions(+), 24 deletions(-) (limited to 'example/ndpi_util.h') diff --git a/example/ndpiReader.c b/example/ndpiReader.c index b0b21b2b3..ca06e98b8 100644 --- a/example/ndpiReader.c +++ b/example/ndpiReader.c @@ -553,17 +553,23 @@ static void printFlow(u_int16_t thread_id, struct ndpi_flow_info *flow) { if(!json_flag) { fprintf(out, "\t%u", ++num_flows); - fprintf(out, "\t%s %s%s%s:%u <-> %s%s%s:%u ", - ipProto2Name(flow->protocol), - (flow->ip_version == 6) ? "[" : "", - flow->lower_name, - (flow->ip_version == 6) ? "]" : "", - ntohs(flow->lower_port), - (flow->ip_version == 6) ? "[" : "", - flow->upper_name, - (flow->ip_version == 6) ? "]" : "", - ntohs(flow->upper_port)); - + fprintf(out, "\t%s ", ipProto2Name(flow->protocol)); + + if(flow->src_to_dst_direction == 1) + fprintf(out, "%s%s%s:%u <-> %s%s%s:%u ", + (flow->ip_version == 6) ? "[" : "", + flow->lower_name, (flow->ip_version == 6) ? "]" : "", ntohs(flow->lower_port), + (flow->ip_version == 6) ? "[" : "", + flow->upper_name, (flow->ip_version == 6) ? "]" : "", ntohs(flow->upper_port) + ); + else + fprintf(out, "%s%s%s:%u <-> %s%s%s:%u ", + (flow->ip_version == 6) ? "[" : "", + flow->upper_name, (flow->ip_version == 6) ? "]" : "", ntohs(flow->upper_port), + (flow->ip_version == 6) ? "[" : "", + flow->lower_name, (flow->ip_version == 6) ? "]" : "", ntohs(flow->lower_port) + ); + if(flow->vlan_id > 0) fprintf(out, "[VLAN: %u]", flow->vlan_id); if(flow->detected_protocol.master_protocol) { diff --git a/example/ndpi_util.c b/example/ndpi_util.c index 783f05c86..445ade1ac 100644 --- a/example/ndpi_util.c +++ b/example/ndpi_util.c @@ -260,9 +260,8 @@ static struct ndpi_flow_info *get_ndpi_flow_info(struct ndpi_workflow * workflow if(iph->protocol == IPPROTO_TCP && l4_packet_len >= 20) { u_int tcp_len; + // tcp workflow->stats.tcp_count++; - - // tcp *tcph = (struct ndpi_tcphdr *)l4; *sport = ntohs((*tcph)->source), *dport = ntohs((*tcph)->dest); @@ -287,12 +286,10 @@ static struct ndpi_flow_info *get_ndpi_flow_info(struct ndpi_workflow * workflow tcp_len = ndpi_min(4*(*tcph)->doff, l4_packet_len); *payload = &l4[tcp_len]; *payload_len = ndpi_max(0, l4_packet_len-4*(*tcph)->doff); - - // udp } else if(iph->protocol == IPPROTO_UDP && l4_packet_len >= 8) { + // udp workflow->stats.udp_count++; - *udph = (struct ndpi_udphdr *)l4; *sport = ntohs((*udph)->source), *dport = ntohs((*udph)->dest); *payload = &l4[sizeof(struct ndpi_udphdr)]; @@ -335,7 +332,9 @@ static struct ndpi_flow_info *get_ndpi_flow_info(struct ndpi_workflow * workflow if(ret == NULL) { if(workflow->stats.ndpi_flow_count == workflow->prefs.max_ndpi_flows) { - NDPI_LOG(0, workflow->ndpi_struct, NDPI_LOG_ERROR, "maximum flow count (%u) has been exceeded\n", workflow->prefs.max_ndpi_flows); + NDPI_LOG(0, workflow->ndpi_struct, NDPI_LOG_ERROR, + "maximum flow count (%u) has been exceeded\n", + workflow->prefs.max_ndpi_flows); exit(-1); } else { struct ndpi_flow_info *newflow = (struct ndpi_flow_info*)malloc(sizeof(struct ndpi_flow_info)); @@ -350,7 +349,8 @@ static struct ndpi_flow_info *get_ndpi_flow_info(struct ndpi_workflow * workflow newflow->lower_ip = lower_ip, newflow->upper_ip = upper_ip; newflow->lower_port = lower_port, newflow->upper_port = upper_port; newflow->ip_version = version; - + newflow->src_to_dst_direction = *src_to_dst_direction; + if(version == IPVERSION) { inet_ntop(AF_INET, &lower_ip, newflow->lower_name, sizeof(newflow->lower_name)); inet_ntop(AF_INET, &upper_ip, newflow->upper_name, sizeof(newflow->upper_name)); @@ -520,7 +520,7 @@ static struct ndpi_proto packet_processing(struct ndpi_workflow * workflow, struct ndpi_udphdr *udph = NULL; u_int16_t sport, dport, payload_len; u_int8_t *payload; - u_int8_t src_to_dst_direction= 1; + u_int8_t src_to_dst_direction = 1; if(iph) flow = get_ndpi_flow_info(workflow, IPVERSION, vlan_id, iph, NULL, @@ -542,9 +542,8 @@ static struct ndpi_proto packet_processing(struct ndpi_workflow * workflow, ndpi_flow = flow->ndpi_flow; flow->packets++, flow->bytes += rawsize; flow->last_seen = time; - } else { + } else return(flow->detected_protocol); - } /* Protocol already detected */ if(flow->detection_completed) return(flow->detected_protocol); @@ -556,8 +555,7 @@ static struct ndpi_proto packet_processing(struct ndpi_workflow * workflow, if((flow->detected_protocol.app_protocol != NDPI_PROTOCOL_UNKNOWN) || ((proto == IPPROTO_UDP) && (flow->packets > 8)) || ((proto == IPPROTO_TCP) && (flow->packets > 10))) { - /* New protocol detected or give up */ - + /* New protocol detected or give up */ flow->detection_completed = 1; } diff --git a/example/ndpi_util.h b/example/ndpi_util.h index a8e21d673..335c94ddf 100644 --- a/example/ndpi_util.h +++ b/example/ndpi_util.h @@ -46,7 +46,7 @@ typedef struct ndpi_flow_info { u_int32_t upper_ip; u_int16_t lower_port; u_int16_t upper_port; - u_int8_t detection_completed, protocol; + u_int8_t detection_completed, protocol, src_to_dst_direction; u_int16_t vlan_id; struct ndpi_flow_struct *ndpi_flow; char lower_name[48], upper_name[48]; -- cgit v1.2.3 From 064b50df819918734062294984e529bd62bd594c Mon Sep 17 00:00:00 2001 From: Luca Deri Date: Sun, 30 Apr 2017 10:12:28 +0200 Subject: Added -m for splitting analysis in sub-analysis steps --- example/ndpiReader.c | 102 ++++++++++++++++++++++++++++++++------------------- example/ndpi_util.c | 2 +- example/ndpi_util.h | 1 + 3 files changed, 67 insertions(+), 38 deletions(-) (limited to 'example/ndpi_util.h') diff --git a/example/ndpiReader.c b/example/ndpiReader.c index 1f982b60b..59ca8b3a1 100644 --- a/example/ndpiReader.c +++ b/example/ndpiReader.c @@ -69,10 +69,12 @@ static u_int8_t live_capture = 0; static u_int8_t undetected_flows_deleted = 0; /** User preferences **/ static u_int8_t enable_protocol_guess = 1, verbose = 0, nDPI_traceLevel = 0, json_flag = 0; +static u_int32_t pcap_analysis_duration = (u_int32_t)-1; static u_int16_t decode_tunnels = 0; static u_int16_t num_loops = 1; static u_int8_t shutdown_app = 0, quiet_mode = 0; static u_int8_t num_threads = 1; +static struct timeval begin, end; #ifdef linux static int core_affinity[MAX_NUM_READER_THREADS]; #endif @@ -146,13 +148,14 @@ static void setupDetection(u_int16_t thread_id, pcap_t * pcap_handle); static void help(u_int long_help) { printf("Welcome to nDPI %s\n\n", ndpi_revision()); - printf("ndpiReader -i [-f ][-s ]\n" + printf("ndpiReader -i [-f ][-s ][-m ]\n" " [-p ][-l [-q][-d][-h][-t][-v ]\n" " [-n ] [-w ] [-j ]\n\n" "Usage:\n" " -i | Specify a pcap file/playlist to read packets from or a device for live capture (comma-separated list)\n" " -f | Specify a BPF filter for filtering selected traffic\n" " -s | Maximum capture duration in seconds (live traffic capture only)\n" + " -m | Split analysis duration in max seconds\n" " -p .protos | Specify a protocol file (eg. protos.txt)\n" " -l | Number of detection loops (test only)\n" " -n | Number of threads. Default: number of interfaces in -i. Ignored with pcap files.\n" @@ -262,7 +265,7 @@ void extcap_config() { int i, argidx = 0; struct ndpi_detection_module_struct *ndpi_mod; struct ndpi_proto_sorter *protos; - + /* -i */ printf("arg {number=%u}{call=-i}{display=Capture Interface or Pcap File Path}{type=string}" "{tooltip=The interface name}\n", argidx++); @@ -271,12 +274,12 @@ void extcap_config() { printf("arg {number=%u}{call=-i}{display=Pcap File to Analize}{type=fileselect}" "{tooltip=The pcap file to analyze (if the interface is unspecified)}\n", argidx++); #endif - + setupDetection(0, NULL); ndpi_mod = ndpi_thread_info[0].workflow->ndpi_struct; - + protos = (struct ndpi_proto_sorter*)malloc(sizeof(struct ndpi_proto_sorter)*ndpi_mod->ndpi_num_supported_protocols); - if(!protos) exit(0); + if(!protos) exit(0); for(i=0; i<(int)ndpi_mod->ndpi_num_supported_protocols; i++) { protos[i].id = i; @@ -284,18 +287,18 @@ void extcap_config() { } qsort(protos, ndpi_mod->ndpi_num_supported_protocols, sizeof(struct ndpi_proto_sorter), cmpProto); - + printf("arg {number=%u}{call=-9}{display=nDPI Protocol Filter}{type=selector}" "{tooltip=nDPI Protocol to be filtered}\n", argidx); printf("value {arg=%d}{value=%d}{display=%s}\n", argidx, -1, "All Protocols (no nDPI filtering)"); - + for(i=0; i<(int)ndpi_mod->ndpi_num_supported_protocols; i++) printf("value {arg=%d}{value=%d}{display=%s (%u)}\n", argidx, protos[i].id, protos[i].name, protos[i].id); free(protos); - + exit(0); } @@ -341,7 +344,7 @@ static void parseOptions(int argc, char **argv) { if(trace) fprintf(trace, " #### %s #### \n", __FUNCTION__); #endif - while ((opt = getopt_long(argc, argv, "df:g:i:hp:l:s:tv:V:n:j:rp:w:q0123:456:7:89:", longopts, &option_idx)) != EOF) { + while ((opt = getopt_long(argc, argv, "df:g:i:hp:l:s:tv:V:n:j:rp:w:q0123:456:7:89:m:", longopts, &option_idx)) != EOF) { #ifdef DEBUG_TRACE if(trace) fprintf(trace, " #### -%c [%s] #### \n", opt, optarg ? optarg : ""); #endif @@ -356,6 +359,10 @@ static void parseOptions(int argc, char **argv) { _pcap_file[0] = optarg; break; + case 'm': + pcap_analysis_duration = atol(optarg); + break; + case 'f': case '6': _bpf_filter = optarg; @@ -580,7 +587,7 @@ static void printFlow(u_int16_t thread_id, struct ndpi_flow_info *flow) { if((verbose != 1) && (verbose != 2)) return; - + if(!json_flag) { fprintf(out, "\t%u", ++num_flows); @@ -937,7 +944,6 @@ static void setupDetection(u_int16_t thread_id, pcap_t * pcap_handle) { * @brief End of detection and free flow */ static void terminateDetection(u_int16_t thread_id) { - ndpi_workflow_free(ndpi_thread_info[thread_id].workflow); } @@ -1047,12 +1053,12 @@ static int port_stats_sort(void *_a, void *_b) { void printPortStats(struct port_stats *stats) { struct port_stats *s, *tmp; int i = 0; - + HASH_ITER(hh, stats, s, tmp) { i++; printf("\t%2d\tPort %5u\t[%u pkts/%u bytes]\n", i, s->port, s->num_pkts, s->num_bytes); if(i >= 10) break; - } + } } /* *********************************************** */ @@ -1061,7 +1067,6 @@ void printPortStats(struct port_stats *stats) { * @brief Print result */ static void printResults(u_int64_t tot_usec) { - u_int32_t i; u_int64_t total_flow_bytes = 0; u_int32_t avg_pkt_size = 0; @@ -1073,10 +1078,10 @@ static void printResults(u_int64_t tot_usec) { json_object *jObj_main = NULL, *jObj_trafficStats, *jArray_detProto = NULL, *jObj; #endif long long unsigned int breed_stats[NUM_BREEDS] = { 0 }; - + memset(&cumulative_stats, 0, sizeof(cumulative_stats)); - for(thread_id = 0; thread_id < num_threads; thread_id++) { + for(thread_id = 0; thread_id < num_threads; thread_id++) { if((ndpi_thread_info[thread_id].workflow->stats.total_wire_bytes == 0) && (ndpi_thread_info[thread_id].workflow->stats.raw_packet_count == 0)) continue; @@ -1090,7 +1095,7 @@ static void printResults(u_int64_t tot_usec) { HASH_SORT(srcStats, port_stats_sort); HASH_SORT(dstStats, port_stats_sort); } - + /* Stats aggregation */ cumulative_stats.guessed_flow_protocols += ndpi_thread_info[thread_id].workflow->stats.guessed_flow_protocols; cumulative_stats.raw_packet_count += ndpi_thread_info[thread_id].workflow->stats.raw_packet_count; @@ -1117,6 +1122,8 @@ static void printResults(u_int64_t tot_usec) { cumulative_stats.max_packet_len += ndpi_thread_info[thread_id].workflow->stats.max_packet_len; } + if(cumulative_stats.total_wire_bytes == 0) return; + if(!quiet_mode) { printf("\nnDPI Memory statistics:\n"); printf("\tnDPI Memory (once): %-13s\n", formatBytes(sizeof(struct ndpi_detection_module_struct), buf, sizeof(buf))); @@ -1155,7 +1162,7 @@ static void printResults(u_int64_t tot_usec) { printf("\tPacket Len > 1500: %-13lu\n", (unsigned long)cumulative_stats.packet_len[5]); if(tot_usec > 0) { - char buf[32], buf1[32]; + char buf[32], buf1[32], when[64]; float t = (float)(cumulative_stats.ip_packet_count*1000000)/(float)tot_usec; float b = (float)(cumulative_stats.total_wire_bytes * 8 *1000000)/(float)tot_usec; float traffic_duration; @@ -1164,6 +1171,11 @@ static void printResults(u_int64_t tot_usec) { printf("\tnDPI throughput: %s pps / %s/sec\n", formatPackets(t, buf), formatTraffic(b, 1, buf1)); t = (float)(cumulative_stats.ip_packet_count*1000000)/(float)traffic_duration; b = (float)(cumulative_stats.total_wire_bytes * 8 *1000000)/(float)traffic_duration; + + strftime(when, sizeof(when), "%d/%b/%Y %H:%M:%S", localtime(&pcap_start.tv_sec)); + printf("\tAnalysis begin: %s\n", when); + strftime(when, sizeof(when), "%d/%b/%Y %H:%M:%S", localtime(&pcap_end.tv_sec)); + printf("\tAnalysis end: %s\n", when); printf("\tTraffic throughput: %s pps / %s/sec\n", formatPackets(t, buf), formatTraffic(b, 1, buf1)); printf("\tTraffic duration: %.3f sec\n", traffic_duration/1000000); } @@ -1315,11 +1327,12 @@ static void printResults(u_int64_t tot_usec) { if(verbose == 3) { printf("\n\nSource Ports Stats:\n"); printPortStats(srcStats); - + printf("\nDestination Ports Stats:\n"); printPortStats(dstStats); - + deletePortsStats(srcStats), deletePortsStats(dstStats); + srcStats = NULL, dstStats = NULL; } } @@ -1328,14 +1341,11 @@ static void printResults(u_int64_t tot_usec) { * @brief Force a pcap_dispatch() or pcap_loop() call to return */ static void breakPcapLoop(u_int16_t thread_id) { - if(ndpi_thread_info[thread_id].workflow->pcap_handle != NULL) { pcap_breakloop(ndpi_thread_info[thread_id].workflow->pcap_handle); } } - - /** * @brief Sigproc is executed for each packet in the pcap file */ @@ -1453,9 +1463,9 @@ static pcap_t * openPcapFileOrDevice(u_int16_t thread_id, const u_char * pcap_fi /** * @brief Check pcap packet */ -static void pcap_packet_callback_checked(u_char *args, - const struct pcap_pkthdr *header, - const u_char *packet) { +static void pcap_process_packet(u_char *args, + const struct pcap_pkthdr *header, + const u_char *packet) { struct ndpi_proto p; u_int16_t thread_id = *((u_int16_t*)args); @@ -1516,13 +1526,13 @@ static void pcap_packet_callback_checked(u_char *args, struct ndpi_packet_trailer *trailer; memcpy(&h, header, sizeof(h)); - + if(h.caplen > (sizeof(extcap_buf)-sizeof(struct ndpi_packet_trailer) - 4)) { printf("INTERNAL ERROR: caplen=%u\n", h.caplen); - h.caplen = sizeof(extcap_buf)-sizeof(struct ndpi_packet_trailer) - 4; + h.caplen = sizeof(extcap_buf)-sizeof(struct ndpi_packet_trailer) - 4; } - trailer = (struct ndpi_packet_trailer*)&extcap_buf[h.caplen]; + trailer = (struct ndpi_packet_trailer*)&extcap_buf[h.caplen]; memcpy(extcap_buf, packet, h.caplen); memset(trailer, 0, sizeof(struct ndpi_packet_trailer)); trailer->magic = htonl(0x19680924); @@ -1546,6 +1556,28 @@ static void pcap_packet_callback_checked(u_char *args, printf("INTERNAL ERROR: ingress packet was modified by nDPI: this should not happen [thread_id=%u, packetId=%lu, caplen=%u]\n", thread_id, (unsigned long)ndpi_thread_info[thread_id].workflow->stats.raw_packet_count, header->caplen); free(packet_checked); + + if((pcap_end.tv_sec-pcap_start.tv_sec) > pcap_analysis_duration) { + int i; + u_int64_t tot_usec; + + gettimeofday(&end, NULL); + tot_usec = end.tv_sec*1000000 + end.tv_usec - (begin.tv_sec*1000000 + begin.tv_usec); + + printResults(tot_usec); + + for(i=0; iprefs.num_roots; i++) { + ndpi_tdestroy(ndpi_thread_info[thread_id].workflow->ndpi_flows_root[i], ndpi_flow_info_freer); + ndpi_thread_info[thread_id].workflow->ndpi_flows_root[i] = NULL; + + memset(&ndpi_thread_info[thread_id].workflow->stats, 0, sizeof(struct ndpi_stats)); + } + + printf("\n-------------------------------------------\n\n"); + + memcpy(&begin, &end, sizeof(begin)); + memcpy(&pcap_start, &pcap_end, sizeof(pcap_start)); + } } @@ -1553,12 +1585,10 @@ static void pcap_packet_callback_checked(u_char *args, * @brief Call pcap_loop() to process packets from a live capture or savefile */ static void runPcapLoop(u_int16_t thread_id) { - if((!shutdown_app) && (ndpi_thread_info[thread_id].workflow->pcap_handle != NULL)) - pcap_loop(ndpi_thread_info[thread_id].workflow->pcap_handle, -1, &pcap_packet_callback_checked, (u_char*)&thread_id); + pcap_loop(ndpi_thread_info[thread_id].workflow->pcap_handle, -1, &pcap_process_packet, (u_char*)&thread_id); } - /** * @brief Process a running thread */ @@ -1583,7 +1613,7 @@ void * processing_thread(void *_thread_id) { if((!json_flag) && (!quiet_mode)) printf("Running thread %ld...\n", thread_id); pcap_loop: - runPcapLoop(thread_id); + runPcapLoop(thread_id); if(playlist_fp[thread_id] != NULL) { /* playlist: read next file */ char filename[256]; @@ -1603,7 +1633,7 @@ void * processing_thread(void *_thread_id) { * @brief Begin, process, end detection process */ void test_lib() { - struct timeval begin, end; + struct timeval end; u_int64_t tot_usec; long thread_id; @@ -1690,8 +1720,6 @@ int main(int argc, char **argv) { automataUnitTest(); memset(ndpi_thread_info, 0, sizeof(ndpi_thread_info)); - memset(&pcap_start, 0, sizeof(pcap_start)); - memset(&pcap_end, 0, sizeof(pcap_end)); parseOptions(argc, argv); @@ -1714,7 +1742,7 @@ int main(int argc, char **argv) { if(results_path) free(results_path); if(results_file) fclose(results_file); if(extcap_dumper) pcap_dump_close(extcap_dumper); - + return 0; } diff --git a/example/ndpi_util.c b/example/ndpi_util.c index eb6744ccc..1ba77eb80 100644 --- a/example/ndpi_util.c +++ b/example/ndpi_util.c @@ -129,7 +129,7 @@ struct ndpi_workflow * ndpi_workflow_init(const struct ndpi_workflow_prefs * pre /* ***************************************************** */ -static void ndpi_flow_info_freer(void *node) { +void ndpi_flow_info_freer(void *node) { struct ndpi_flow_info *flow = (struct ndpi_flow_info*)node; ndpi_free_flow_info_half(flow); diff --git a/example/ndpi_util.h b/example/ndpi_util.h index 335c94ddf..9d8f1e446 100644 --- a/example/ndpi_util.h +++ b/example/ndpi_util.h @@ -161,4 +161,5 @@ static inline void ndpi_workflow_set_flow_giveup_callback(struct ndpi_workflow * int ndpi_workflow_node_cmp(const void *a, const void *b); void process_ndpi_collected_info(struct ndpi_workflow * workflow, struct ndpi_flow_info *flow); void ethernet_crc32(const void* data, size_t n_bytes, uint32_t* crc); +void ndpi_flow_info_freer(void *node); #endif -- cgit v1.2.3 From 4030b52ecb5c7a98eeb3eadc362eab60521fb565 Mon Sep 17 00:00:00 2001 From: berat Date: Mon, 1 May 2017 14:07:41 +0200 Subject: Added IP Statistics to Port Stats --- example/ndpiReader.c | 179 ++++++++++++++++++++++++++++++++++++++++++++++++--- example/ndpi_util.h | 3 +- 2 files changed, 172 insertions(+), 10 deletions(-) (limited to 'example/ndpi_util.h') diff --git a/example/ndpiReader.c b/example/ndpiReader.c index 59ca8b3a1..63e33d783 100644 --- a/example/ndpiReader.c +++ b/example/ndpiReader.c @@ -84,9 +84,24 @@ static time_t capture_for = 0; static time_t capture_until = 0; static u_int32_t num_flows; +struct info_pair{ + u_int32_t addr; + int count; +}; + +typedef struct node_a{ + u_int32_t addr; + int count; + struct node_a *left, *right; +}addr_node; + struct port_stats { u_int32_t port; /* we'll use this field as the key */ u_int32_t num_pkts, num_bytes; + u_int32_t num_addr; /*to hold number of distinct IP addresses*/ + u_int32_t cumulative_addr; /*to hold cumulative some of IP addresses*/ + addr_node *addr_tree; /* to hold distinct IP addresses*/ + struct info_pair top_ip_addrs[MAX_NUM_IP_ADDRESS]; UT_hash_handle hh; /* makes this structure hashable */ }; @@ -136,6 +151,9 @@ FILE *trace = NULL; /********************** FUNCTIONS ********************* */ + + + /** * @brief Set main components necessary to the detection */ @@ -764,8 +782,99 @@ static void node_proto_guess_walker(const void *node, ndpi_VISIT which, int dept /* *********************************************** */ -static void updatePortStats(struct port_stats **stats, u_int32_t port, u_int32_t num_pkts, u_int32_t num_bytes) { +int updateIpTree(const u_int32_t key, addr_node **vrootp){ + addr_node *q; + addr_node **rootp = vrootp; + + if(rootp == (addr_node **)0) + return 0; + + while (*rootp != (addr_node *)0) { /* Knuth's T1: */ + + if(key == ((*rootp)->addr)) { /* T2: */ + return ++((*rootp)->count); + } + + rootp = (key < ((*rootp)->addr)) ? + &(*rootp)->left : /* T3: follow left branch */ + &(*rootp)->right; /* T4: follow right branch */ + } + q = (addr_node *) malloc(sizeof(addr_node)); /* T5: key not found */ + if(q != (addr_node *)0) { /* make new node */ + *rootp = q; /* link new node to old */ + q->addr = key; /* initialize new node */ + q->count = 1; + q->left = q->right = (addr_node *)0; + return q->count; + } + +} + +/* *********************************************** */ + +void freeIpTree(addr_node *root) { + while (root != NULL) { + addr_node *left = root->left; + if (left == NULL) { + addr_node *right = root->right; + root->right = NULL; + root = right; + } else { + /* Rotate the left child up.*/ + root->left = left->right; + left->right = root; + root = left; + } + } +} + +/* *********************************************** */ + +void updateTopIpAddress(u_int32_t addr, int count, struct info_pair top[], int size){ + int update = 0; + int min_i = 0; + int min = count; + + if(count == 0) return; + + struct info_pair pair; + pair.addr = addr, pair.count = count; + + /* if the same ip with a bigger + count just update it */ + for(int i=0; iport = port, s->num_pkts = 0, s->num_bytes = 0; + s->num_addr = 1, s->cumulative_addr = 1; + + memset(s->top_ip_addrs, 0, MAX_NUM_IP_ADDRESS*sizeof(struct info_pair)); + updateTopIpAddress(addr, 1, s->top_ip_addrs, MAX_NUM_IP_ADDRESS); + + s->addr_tree = (addr_node *) malloc(sizeof(addr_node)); + if(!s->addr_tree) return; + + s->addr_tree->addr = addr; + s->addr_tree->count = 1; + s->addr_tree->left = NULL; + s->addr_tree->right = NULL; + HASH_ADD_INT(*stats, port, s); } + int count = updateIpTree(addr, &(*s).addr_tree); + if(count == UPDATED_TREE) s->num_addr++; + if(count) { + s->cumulative_addr++; + updateTopIpAddress(addr, count, s->top_ip_addrs, MAX_NUM_IP_ADDRESS); + } + s->num_pkts += num_pkts, s->num_bytes += num_bytes; + } /* *********************************************** */ @@ -786,6 +916,8 @@ static void deletePortsStats(struct port_stats *stats) { HASH_ITER(hh, stats, current_port, tmp) { HASH_DEL(stats, current_port); + freeIpTree(current_port->addr_tree); + free(current_port->addr_tree); free(current_port); } } @@ -798,14 +930,18 @@ static void deletePortsStats(struct port_stats *stats) { static void port_stats_walker(const void *node, ndpi_VISIT which, int depth, void *user_data) { struct ndpi_flow_info *flow = *(struct ndpi_flow_info **) node; u_int16_t sport, dport; + u_int32_t saddr, daddr; - if(flow->src_to_dst_direction == 1) - sport = ntohs(flow->lower_port), dport = ntohs(flow->upper_port); - else - sport = ntohs(flow->upper_port), dport = ntohs(flow->lower_port); - - updatePortStats(&srcStats, sport, flow->packets, flow->bytes); - updatePortStats(&dstStats, dport, flow->packets, flow->bytes); + if(flow->src_to_dst_direction == 1) { + sport = ntohs(flow->lower_port), dport = ntohs(flow->upper_port); + saddr = flow->lower_ip, daddr = flow->upper_ip; + } + else { + sport = ntohs(flow->upper_port), dport = ntohs(flow->lower_port); + saddr = flow->upper_ip, daddr = flow->lower_ip; + } + updatePortStats(&srcStats, sport, saddr, flow->packets, flow->bytes); + updatePortStats(&dstStats, dport, daddr, flow->packets, flow->bytes); } /* *********************************************** */ @@ -1050,13 +1186,38 @@ static int port_stats_sort(void *_a, void *_b) { /* *********************************************** */ +static int info_pair_cmp (const void *_a, const void *_b) +{ + struct info_pair *a = (struct info_pair *)_a; + struct info_pair *b = (struct info_pair *)_b; + return b->count - a->count; +} + +/* *********************************************** */ + void printPortStats(struct port_stats *stats) { struct port_stats *s, *tmp; + char ip_name[48]; int i = 0; + int first = 1; + HASH_ITER(hh, stats, s, tmp) { i++; - printf("\t%2d\tPort %5u\t[%u pkts/%u bytes]\n", i, s->port, s->num_pkts, s->num_bytes); + printf("\t%2d\tPort %5u\t[%u IP address/%u pkts/%u bytes]\n", i, s->port, s->num_addr, s->num_pkts, s->num_bytes); + + qsort(&s->top_ip_addrs[0], MAX_NUM_IP_ADDRESS, sizeof(struct info_pair), info_pair_cmp); + + for(int i=0;itop_ip_addrs[i].count != 0) { + inet_ntop(AF_INET, &s->top_ip_addrs[i].addr, ip_name, sizeof(ip_name)); + printf("\t\t\t\t%s\t%s ~ %.2f%%\n", (first) ? "Top IP Stats:" : "\t", + ip_name, ((s->top_ip_addrs[i].count) * 100.0) / s->cumulative_addr); + first = 0; + } + } + printf("\n"); + first = 1; if(i >= 10) break; } } diff --git a/example/ndpi_util.h b/example/ndpi_util.h index 9d8f1e446..ca9f20274 100644 --- a/example/ndpi_util.h +++ b/example/ndpi_util.h @@ -38,7 +38,8 @@ #define NUM_ROOTS 512 #define MAX_NDPI_FLOWS 200000000 #define TICK_RESOLUTION 1000 - +#define MAX_NUM_IP_ADDRESS 5 /* len of ip address array */ +#define UPDATED_TREE 1 // flow tracking typedef struct ndpi_flow_info { -- cgit v1.2.3