From 4ba106a525e1a93539decca3cad1e4ce38a2d48b Mon Sep 17 00:00:00 2001 From: emanuele-f Date: Tue, 19 Apr 2016 19:25:42 +0200 Subject: Change confusing "ndpi_flow" structure name to "ndpi_flow_info" --- example/ndpiReader.c | 58 ++++++++++++++++++++++++++-------------------------- 1 file changed, 29 insertions(+), 29 deletions(-) (limited to 'example/ndpiReader.c') diff --git a/example/ndpiReader.c b/example/ndpiReader.c index c678fe1dd..cd2dc1b3d 100644 --- a/example/ndpiReader.c +++ b/example/ndpiReader.c @@ -164,7 +164,7 @@ struct reader_thread { /* TODO Add barrier */ struct thread_stats stats; - struct ndpi_flow *idle_flows[IDLE_SCAN_BUDGET]; + struct ndpi_flow_info *idle_flows[IDLE_SCAN_BUDGET]; }; static struct reader_thread ndpi_thread_info[MAX_NUM_READER_THREADS]; @@ -180,7 +180,7 @@ typedef struct ndpi_id { static u_int32_t size_id_struct = 0; // ID tracking structure size // flow tracking -typedef struct ndpi_flow { +typedef struct ndpi_flow_info { u_int32_t lower_ip; u_int32_t upper_ip; u_int16_t lower_port; @@ -205,7 +205,7 @@ typedef struct ndpi_flow { } ssl; void *src_id, *dst_id; -} ndpi_flow_t; +} ndpi_flow_info_t; static u_int32_t size_flow_struct = 0; @@ -489,7 +489,7 @@ char* intoaV4(unsigned int addr, char* buf, u_short bufLen) { /* ***************************************************** */ -static void printFlow(u_int16_t thread_id, struct ndpi_flow *flow) { +static void printFlow(u_int16_t thread_id, struct ndpi_flow_info *flow) { #ifdef HAVE_JSON_C json_object *jObj; #endif @@ -590,7 +590,7 @@ static void printFlow(u_int16_t thread_id, struct ndpi_flow *flow) { /* ***************************************************** */ -static void free_ndpi_flow(struct ndpi_flow *flow) { +static void free_ndpi_flow_info(struct ndpi_flow_info *flow) { if(flow->ndpi_flow) { ndpi_free_flow(flow->ndpi_flow); flow->ndpi_flow = NULL; } if(flow->src_id) { ndpi_free(flow->src_id); flow->src_id = NULL; } if(flow->dst_id) { ndpi_free(flow->dst_id); flow->dst_id = NULL; } @@ -599,17 +599,17 @@ static void free_ndpi_flow(struct ndpi_flow *flow) { /* ***************************************************** */ -static void ndpi_flow_freer(void *node) { - struct ndpi_flow *flow = (struct ndpi_flow*)node; +static void ndpi_flow_info_freer(void *node) { + struct ndpi_flow_info *flow = (struct ndpi_flow_info*)node; - free_ndpi_flow(flow); + free_ndpi_flow_info(flow); ndpi_free(flow); } /* ***************************************************** */ static void node_print_unknown_proto_walker(const void *node, ndpi_VISIT which, int depth, void *user_data) { - struct ndpi_flow *flow = *(struct ndpi_flow**)node; + struct ndpi_flow_info *flow = *(struct ndpi_flow_info**)node; u_int16_t thread_id = *((u_int16_t*)user_data); if(flow->detected_protocol.protocol != NDPI_PROTOCOL_UNKNOWN) return; @@ -621,7 +621,7 @@ static void node_print_unknown_proto_walker(const void *node, ndpi_VISIT which, /* ***************************************************** */ static void node_print_known_proto_walker(const void *node, ndpi_VISIT which, int depth, void *user_data) { - struct ndpi_flow *flow = *(struct ndpi_flow**)node; + struct ndpi_flow_info *flow = *(struct ndpi_flow_info**)node; u_int16_t thread_id = *((u_int16_t*)user_data); if(flow->detected_protocol.protocol == NDPI_PROTOCOL_UNKNOWN) return; @@ -632,7 +632,7 @@ static void node_print_known_proto_walker(const void *node, ndpi_VISIT which, in /* ***************************************************** */ -static u_int16_t node_guess_undetected_protocol(u_int16_t thread_id, struct ndpi_flow *flow) { +static u_int16_t node_guess_undetected_protocol(u_int16_t thread_id, struct ndpi_flow_info *flow) { flow->detected_protocol = ndpi_guess_undetected_protocol(ndpi_thread_info[thread_id].ndpi_struct, flow->protocol, ntohl(flow->lower_ip), @@ -649,7 +649,7 @@ static u_int16_t node_guess_undetected_protocol(u_int16_t thread_id, struct ndpi /* ***************************************************** */ static void node_proto_guess_walker(const void *node, ndpi_VISIT which, int depth, void *user_data) { - struct ndpi_flow *flow = *(struct ndpi_flow **) node; + struct ndpi_flow_info *flow = *(struct ndpi_flow_info **) node; u_int16_t thread_id = *((u_int16_t *) user_data); if((which == ndpi_preorder) || (which == ndpi_leaf)) { /* Avoid walking the same node multiple times */ @@ -672,7 +672,7 @@ static void node_proto_guess_walker(const void *node, ndpi_VISIT which, int dept /* ***************************************************** */ static void node_idle_scan_walker(const void *node, ndpi_VISIT which, int depth, void *user_data) { - struct ndpi_flow *flow = *(struct ndpi_flow **) node; + struct ndpi_flow_info *flow = *(struct ndpi_flow_info **) node; u_int16_t thread_id = *((u_int16_t *) user_data); if(ndpi_thread_info[thread_id].num_idle_flows == IDLE_SCAN_BUDGET) /* TODO optimise with a budget-based walk */ @@ -687,7 +687,7 @@ static void node_idle_scan_walker(const void *node, ndpi_VISIT which, int depth, if((flow->detected_protocol.protocol == NDPI_PROTOCOL_UNKNOWN) && !undetected_flows_deleted) undetected_flows_deleted = 1; - free_ndpi_flow(flow); + free_ndpi_flow_info(flow); ndpi_thread_info[thread_id].stats.ndpi_flow_count--; /* adding to a queue (we can't delete it from the tree inline ) */ @@ -699,8 +699,8 @@ static void node_idle_scan_walker(const void *node, ndpi_VISIT which, int depth, /* ***************************************************** */ static int node_cmp(const void *a, const void *b) { - struct ndpi_flow *fa = (struct ndpi_flow*)a; - struct ndpi_flow *fb = (struct ndpi_flow*)b; + struct ndpi_flow_info *fa = (struct ndpi_flow_info*)a; + struct ndpi_flow_info *fb = (struct ndpi_flow_info*)b; if(fa->vlan_id < fb->vlan_id ) return(-1); else { if(fa->vlan_id > fb->vlan_id ) return(1); } if(fa->lower_ip < fb->lower_ip ) return(-1); else { if(fa->lower_ip > fb->lower_ip ) return(1); } @@ -714,7 +714,7 @@ static int node_cmp(const void *a, const void *b) { /* ***************************************************** */ -static struct ndpi_flow *get_ndpi_flow(u_int16_t thread_id, +static struct ndpi_flow_info *get_ndpi_flow_info(u_int16_t thread_id, const u_int8_t version, u_int16_t vlan_id, const struct ndpi_iphdr *iph, @@ -736,7 +736,7 @@ static struct ndpi_flow *get_ndpi_flow(u_int16_t thread_id, u_int32_t upper_ip; u_int16_t lower_port; u_int16_t upper_port; - struct ndpi_flow flow; + struct ndpi_flow_info flow; void *ret; u_int8_t *l3, *l4; @@ -866,14 +866,14 @@ static struct ndpi_flow *get_ndpi_flow(u_int16_t thread_id, printf("ERROR: maximum flow count (%u) has been exceeded\n", MAX_NDPI_FLOWS); exit(-1); } else { - struct ndpi_flow *newflow = (struct ndpi_flow*)malloc(sizeof(struct ndpi_flow)); + struct ndpi_flow_info *newflow = (struct ndpi_flow_info*)malloc(sizeof(struct ndpi_flow_info)); if(newflow == NULL) { printf("[NDPI] %s(1): not enough memory\n", __FUNCTION__); return(NULL); } - memset(newflow, 0, sizeof(struct ndpi_flow)); + memset(newflow, 0, sizeof(struct ndpi_flow_info)); newflow->protocol = iph->protocol, newflow->vlan_id = vlan_id; newflow->lower_ip = lower_ip, newflow->upper_ip = upper_ip; newflow->lower_port = lower_port, newflow->upper_port = upper_port; @@ -917,7 +917,7 @@ static struct ndpi_flow *get_ndpi_flow(u_int16_t thread_id, return newflow; } } else { - struct ndpi_flow *flow = *(struct ndpi_flow**)ret; + struct ndpi_flow_info *flow = *(struct ndpi_flow_info**)ret; if(flow->lower_ip == lower_ip && flow->upper_ip == upper_ip && flow->lower_port == lower_port && flow->upper_port == upper_port) @@ -931,7 +931,7 @@ static struct ndpi_flow *get_ndpi_flow(u_int16_t thread_id, /* ***************************************************** */ -static struct ndpi_flow *get_ndpi_flow6(u_int16_t thread_id, +static struct ndpi_flow_info *get_ndpi_flow_info6(u_int16_t thread_id, u_int16_t vlan_id, const struct ndpi_ipv6hdr *iph6, u_int16_t ip_offset, @@ -958,7 +958,7 @@ static struct ndpi_flow *get_ndpi_flow6(u_int16_t thread_id, iph.protocol = options[0]; } - return(get_ndpi_flow(thread_id, 6, vlan_id, &iph, iph6, ip_offset, + return(get_ndpi_flow_info(thread_id, 6, vlan_id, &iph, iph6, ip_offset, sizeof(struct ndpi_ipv6hdr), ntohs(iph6->ip6_ctlun.ip6_un1.ip6_un1_plen), tcph, udph, sport, dport, @@ -1005,7 +1005,7 @@ static void terminateDetection(u_int16_t thread_id) { int i; for(i=0; itot_len) - (iph->ihl * 4), &tcph, &udph, &sport, &dport, &src, &dst, &proto, &payload, &payload_len, &src_to_dst_direction); else - flow = get_ndpi_flow6(thread_id, vlan_id, iph6, ip_offset, + flow = get_ndpi_flow_info6(thread_id, vlan_id, iph6, ip_offset, &tcph, &udph, &sport, &dport, &src, &dst, &proto, &payload, &payload_len, &src_to_dst_direction); @@ -1090,7 +1090,7 @@ static unsigned int packet_processing(u_int16_t thread_id, if(flow->detected_protocol.protocol == NDPI_PROTOCOL_UNKNOWN) flow->detected_protocol = ndpi_detection_giveup(ndpi_thread_info[thread_id].ndpi_struct, flow->ndpi_flow); - free_ndpi_flow(flow); + free_ndpi_flow_info(flow); if(verbose > 1) { if(enable_protocol_guess) { @@ -1116,7 +1116,7 @@ static unsigned int packet_processing(u_int16_t thread_id, ndpi_tdelete(ndpi_thread_info[thread_id].idle_flows[--ndpi_thread_info[thread_id].num_idle_flows], &ndpi_thread_info[thread_id].ndpi_flows_root[ndpi_thread_info[thread_id].idle_scan_idx], node_cmp); /* free the memory associated to idle flow in "idle_flows" - (see struct reader thread)*/ - free_ndpi_flow(ndpi_thread_info[thread_id].idle_flows[ndpi_thread_info[thread_id].num_idle_flows]); + free_ndpi_flow_info(ndpi_thread_info[thread_id].idle_flows[ndpi_thread_info[thread_id].num_idle_flows]); ndpi_free(ndpi_thread_info[thread_id].idle_flows[ndpi_thread_info[thread_id].num_idle_flows]); } -- cgit v1.2.3 From a4d0af1f96bba36f1f98c1090418d49b94ac319a Mon Sep 17 00:00:00 2001 From: emanuele-f Date: Tue, 19 Apr 2016 22:35:18 +0200 Subject: Move relevant functions to ndpi_util API module --- example/ndpiReader.c | 1121 ++++++----------------------------------------- src/include/ndpi_util.h | 116 +++++ src/lib/ndpi_util.c | 864 ++++++++++++++++++++++++++++++++++++ 3 files changed, 1111 insertions(+), 990 deletions(-) create mode 100644 src/include/ndpi_util.h create mode 100644 src/lib/ndpi_util.c (limited to 'example/ndpiReader.c') diff --git a/example/ndpiReader.c b/example/ndpiReader.c index cd2dc1b3d..c2cfcf33d 100644 --- a/example/ndpiReader.c +++ b/example/ndpiReader.c @@ -56,42 +56,15 @@ #define MAX_IDLE_TIME 30000 #define IDLE_SCAN_BUDGET 1024 #define NUM_ROOTS 512 -#define GTP_U_V1_PORT 2152 -#define TZSP_PORT 37008 #define MAX_NDPI_FLOWS 200000000 -#ifndef ETH_P_IP -#define ETH_P_IP 0x0800 /* IPv4 */ -#endif - -#ifndef ETH_P_IPv6 -#define ETH_P_IPV6 0x86dd /* IPv6 */ -#endif - -#define SLARP 0x8035 /* Cisco Slarp */ -#define CISCO_D_PROTO 0x2000 /* Cisco Discovery Protocol */ - -#define VLAN 0x8100 -#define MPLS_UNI 0x8847 -#define MPLS_MULTI 0x8848 -#define PPPoE 0x8864 -#define SNAP 0xaa - -/* mask for FCF */ -#define WIFI_DATA 0x2 /* 0000 0010 */ -#define FCF_TYPE(fc) (((fc) >> 2) & 0x3) /* 0000 0011 = 0x3 */ -#define FCF_SUBTYPE(fc) (((fc) >> 4) & 0xF) /* 0000 1111 = 0xF */ -#define FCF_TO_DS(fc) ((fc) & 0x0100) -#define FCF_FROM_DS(fc) ((fc) & 0x0200) - -/* mask for Bad FCF presence */ -#define BAD_FCS 0x50 /* 0101 0000 */ +#include "ndpi_util.h" /** * @brief Set main components necessary to the detection * @details TODO */ -static void setupDetection(u_int16_t thread_id); +static void setupDetection(u_int16_t thread_id, pcap_t * pcap_handle); /** * Client parameters @@ -118,7 +91,6 @@ static u_int16_t decode_tunnels = 0; static u_int16_t num_loops = 1; static u_int8_t shutdown_app = 0, quiet_mode = 0; static u_int8_t num_threads = 1; -static u_int32_t current_ndpi_memory = 0, max_ndpi_memory = 0; #ifdef linux static int core_affinity[MAX_NUM_READER_THREADS]; #endif @@ -134,37 +106,10 @@ static time_t capture_until = 0; static u_int32_t num_flows; -struct thread_stats { - u_int32_t guessed_flow_protocols; - u_int64_t raw_packet_count; - u_int64_t ip_packet_count; - u_int64_t total_wire_bytes, total_ip_bytes, total_discarded_bytes; - u_int64_t protocol_counter[NDPI_MAX_SUPPORTED_PROTOCOLS + NDPI_MAX_NUM_CUSTOM_PROTOCOLS + 1]; - u_int64_t protocol_counter_bytes[NDPI_MAX_SUPPORTED_PROTOCOLS + NDPI_MAX_NUM_CUSTOM_PROTOCOLS + 1]; - u_int32_t protocol_flows[NDPI_MAX_SUPPORTED_PROTOCOLS + NDPI_MAX_NUM_CUSTOM_PROTOCOLS + 1]; - u_int32_t ndpi_flow_count; - u_int64_t tcp_count, udp_count; - u_int64_t mpls_count, pppoe_count, vlan_count, fragmented_count; - u_int64_t packet_len[6]; - u_int16_t max_packet_len; -}; - struct reader_thread { - struct ndpi_detection_module_struct *ndpi_struct; - void *ndpi_flows_root[NUM_ROOTS]; - char _pcap_error_buffer[PCAP_ERRBUF_SIZE]; - pcap_t *_pcap_handle; - u_int64_t last_time; - u_int64_t last_idle_scan_time; - u_int32_t idle_scan_idx; - u_int32_t num_idle_flows; + struct ndpi_workflow * workflow; pthread_t pthread; - int _pcap_datalink_type; - - /* TODO Add barrier */ - struct thread_stats stats; - - struct ndpi_flow_info *idle_flows[IDLE_SCAN_BUDGET]; + pcap_t * pcap_handle; }; static struct reader_thread ndpi_thread_info[MAX_NUM_READER_THREADS]; @@ -179,34 +124,6 @@ typedef struct ndpi_id { static u_int32_t size_id_struct = 0; // ID tracking structure size -// flow tracking -typedef struct ndpi_flow_info { - u_int32_t lower_ip; - u_int32_t upper_ip; - u_int16_t lower_port; - u_int16_t upper_port; - u_int8_t detection_completed, protocol; - u_int16_t vlan_id; - struct ndpi_flow_struct *ndpi_flow; - char lower_name[48], upper_name[48]; - u_int8_t ip_version; - u_int64_t last_seen; - u_int64_t bytes; - u_int32_t packets; - - // result only, not used for flow identification - ndpi_protocol detected_protocol; - - char host_server_name[192]; - char bittorent_hash[41]; - - struct { - char client_certificate[48], server_certificate[48]; - } ssl; - - void *src_id, *dst_id; -} ndpi_flow_info_t; - static u_int32_t size_flow_struct = 0; @@ -237,8 +154,8 @@ static void help(u_int long_help) { if(long_help) { printf("\n\nSupported protocols:\n"); num_threads = 1; - setupDetection(0); - ndpi_dump_protocols(ndpi_thread_info[0].ndpi_struct); + setupDetection(0, NULL); + ndpi_dump_protocols(ndpi_thread_info[0].workflow->ndpi_struct); } exit(!long_help); @@ -369,62 +286,6 @@ static void parseOptions(int argc, char **argv) { #endif } -/* ***************************************************** */ - -static void debug_printf(u_int32_t protocol, void *id_struct, - ndpi_log_level_t log_level, - const char *format, ...) { - va_list va_ap; -#ifndef WIN32 - struct tm result; -#endif - - if(log_level <= nDPI_traceLevel) { - char buf[8192], out_buf[8192]; - char theDate[32]; - const char *extra_msg = ""; - time_t theTime = time(NULL); - - va_start (va_ap, format); - - if(log_level == NDPI_LOG_ERROR) - extra_msg = "ERROR: "; - else if(log_level == NDPI_LOG_TRACE) - extra_msg = "TRACE: "; - else - extra_msg = "DEBUG: "; - - memset(buf, 0, sizeof(buf)); - strftime(theDate, 32, "%d/%b/%Y %H:%M:%S", localtime_r(&theTime,&result) ); - vsnprintf(buf, sizeof(buf)-1, format, va_ap); - - snprintf(out_buf, sizeof(out_buf), "%s %s%s", theDate, extra_msg, buf); - printf("%s", out_buf); - fflush(stdout); - } - - va_end(va_ap); -} - -/* ***************************************************** */ - -static void *malloc_wrapper(size_t size) { - current_ndpi_memory += size; - - if(current_ndpi_memory > max_ndpi_memory) - max_ndpi_memory = current_ndpi_memory; - - return malloc(size); -} - -/* ***************************************************** */ - -static void free_wrapper(void *freeable) { - free(freeable); -} - -/* ***************************************************** */ - static char* ipProto2Name(u_short proto_id) { static char proto[8]; @@ -487,6 +348,24 @@ char* intoaV4(unsigned int addr, char* buf, u_short bufLen) { return(retStr); } +/* ***************************************************** */ +static u_int32_t current_ndpi_memory = 0, max_ndpi_memory = 0; + +static void *malloc_wrapper(size_t size) { + current_ndpi_memory += size; + + if(current_ndpi_memory > max_ndpi_memory) + max_ndpi_memory = current_ndpi_memory; + + return malloc(size); +} + +/* ***************************************************** */ + +static void free_wrapper(void *freeable) { + free(freeable); +} + /* ***************************************************** */ static void printFlow(u_int16_t thread_id, struct ndpi_flow_info *flow) { @@ -516,12 +395,12 @@ static void printFlow(u_int16_t thread_id, struct ndpi_flow_info *flow) { fprintf(out, "[proto: %u.%u/%s]", flow->detected_protocol.master_protocol, flow->detected_protocol.protocol, - ndpi_protocol2name(ndpi_thread_info[thread_id].ndpi_struct, + ndpi_protocol2name(ndpi_thread_info[thread_id].workflow->ndpi_struct, flow->detected_protocol, buf, sizeof(buf))); } else fprintf(out, "[proto: %u/%s]", flow->detected_protocol.protocol, - ndpi_get_proto_name(ndpi_thread_info[thread_id].ndpi_struct, flow->detected_protocol.protocol)); + ndpi_get_proto_name(ndpi_thread_info[thread_id].workflow->ndpi_struct, flow->detected_protocol.protocol)); fprintf(out, "[%u pkts/%llu bytes]", flow->packets, (long long unsigned int)flow->bytes); @@ -551,14 +430,14 @@ static void printFlow(u_int16_t thread_id, struct ndpi_flow_info *flow) { char tmp[256]; snprintf(tmp, sizeof(tmp), "%s.%s", - ndpi_get_proto_name(ndpi_thread_info[thread_id].ndpi_struct, flow->detected_protocol.master_protocol), - ndpi_get_proto_name(ndpi_thread_info[thread_id].ndpi_struct, flow->detected_protocol.protocol)); + ndpi_get_proto_name(ndpi_thread_info[thread_id].workflow->ndpi_struct, flow->detected_protocol.master_protocol), + ndpi_get_proto_name(ndpi_thread_info[thread_id].workflow->ndpi_struct, flow->detected_protocol.protocol)); json_object_object_add(jObj,"detected.protocol.name", json_object_new_string(tmp)); } else json_object_object_add(jObj,"detected.protocol.name", - json_object_new_string(ndpi_get_proto_name(ndpi_thread_info[thread_id].ndpi_struct, + json_object_new_string(ndpi_get_proto_name(ndpi_thread_info[thread_id].workflow->ndpi_struct, flow->detected_protocol.protocol))); json_object_object_add(jObj,"packets",json_object_new_int(flow->packets)); @@ -633,7 +512,7 @@ static void node_print_known_proto_walker(const void *node, ndpi_VISIT which, in /* ***************************************************** */ static u_int16_t node_guess_undetected_protocol(u_int16_t thread_id, struct ndpi_flow_info *flow) { - flow->detected_protocol = ndpi_guess_undetected_protocol(ndpi_thread_info[thread_id].ndpi_struct, + flow->detected_protocol = ndpi_guess_undetected_protocol(ndpi_thread_info[thread_id].workflow->ndpi_struct, flow->protocol, ntohl(flow->lower_ip), ntohs(flow->lower_port), @@ -641,7 +520,7 @@ static u_int16_t node_guess_undetected_protocol(u_int16_t thread_id, struct ndpi ntohs(flow->upper_port)); // printf("Guess state: %u\n", flow->detected_protocol); if(flow->detected_protocol.protocol != NDPI_PROTOCOL_UNKNOWN) - ndpi_thread_info[thread_id].stats.guessed_flow_protocols++; + ndpi_thread_info[thread_id].workflow->stats.guessed_flow_protocols++; return(flow->detected_protocol.protocol); } @@ -654,7 +533,7 @@ static void node_proto_guess_walker(const void *node, ndpi_VISIT which, int dept if((which == ndpi_preorder) || (which == ndpi_leaf)) { /* Avoid walking the same node multiple times */ if((!flow->detection_completed) && flow->ndpi_flow) - flow->detected_protocol = ndpi_detection_giveup(ndpi_thread_info[0].ndpi_struct, flow->ndpi_flow); + flow->detected_protocol = ndpi_detection_giveup(ndpi_thread_info[0].workflow->ndpi_struct, flow->ndpi_flow); if(enable_protocol_guess) { if(flow->detected_protocol.protocol == NDPI_PROTOCOL_UNKNOWN) { @@ -663,9 +542,9 @@ static void node_proto_guess_walker(const void *node, ndpi_VISIT which, int dept } } - ndpi_thread_info[thread_id].stats.protocol_counter[flow->detected_protocol.protocol] += flow->packets; - ndpi_thread_info[thread_id].stats.protocol_counter_bytes[flow->detected_protocol.protocol] += flow->bytes; - ndpi_thread_info[thread_id].stats.protocol_flows[flow->detected_protocol.protocol]++; + ndpi_thread_info[thread_id].workflow->stats.protocol_counter[flow->detected_protocol.protocol] += flow->packets; + ndpi_thread_info[thread_id].workflow->stats.protocol_counter_bytes[flow->detected_protocol.protocol] += flow->bytes; + ndpi_thread_info[thread_id].workflow->stats.protocol_flows[flow->detected_protocol.protocol]++; } } @@ -675,11 +554,11 @@ static void node_idle_scan_walker(const void *node, ndpi_VISIT which, int depth, struct ndpi_flow_info *flow = *(struct ndpi_flow_info **) node; u_int16_t thread_id = *((u_int16_t *) user_data); - if(ndpi_thread_info[thread_id].num_idle_flows == IDLE_SCAN_BUDGET) /* TODO optimise with a budget-based walk */ + if(ndpi_thread_info[thread_id].workflow->num_idle_flows == IDLE_SCAN_BUDGET) /* TODO optimise with a budget-based walk */ return; if((which == ndpi_preorder) || (which == ndpi_leaf)) { /* Avoid walking the same node multiple times */ - if(flow->last_seen + MAX_IDLE_TIME < ndpi_thread_info[thread_id].last_time) { + if(flow->last_seen + MAX_IDLE_TIME < ndpi_thread_info[thread_id].workflow->last_time) { /* update stats */ node_proto_guess_walker(node, which, depth, user_data); @@ -688,315 +567,42 @@ static void node_idle_scan_walker(const void *node, ndpi_VISIT which, int depth, undetected_flows_deleted = 1; free_ndpi_flow_info(flow); - ndpi_thread_info[thread_id].stats.ndpi_flow_count--; + ndpi_thread_info[thread_id].workflow->stats.ndpi_flow_count--; /* adding to a queue (we can't delete it from the tree inline ) */ - ndpi_thread_info[thread_id].idle_flows[ndpi_thread_info[thread_id].num_idle_flows++] = flow; + ndpi_thread_info[thread_id].workflow->idle_flows[ndpi_thread_info[thread_id].workflow->num_idle_flows++] = flow; } } } /* ***************************************************** */ -static int node_cmp(const void *a, const void *b) { - struct ndpi_flow_info *fa = (struct ndpi_flow_info*)a; - struct ndpi_flow_info *fb = (struct ndpi_flow_info*)b; - - if(fa->vlan_id < fb->vlan_id ) return(-1); else { if(fa->vlan_id > fb->vlan_id ) return(1); } - if(fa->lower_ip < fb->lower_ip ) return(-1); else { if(fa->lower_ip > fb->lower_ip ) return(1); } - if(fa->lower_port < fb->lower_port) return(-1); else { if(fa->lower_port > fb->lower_port) return(1); } - if(fa->upper_ip < fb->upper_ip ) return(-1); else { if(fa->upper_ip > fb->upper_ip ) return(1); } - if(fa->upper_port < fb->upper_port) return(-1); else { if(fa->upper_port > fb->upper_port) return(1); } - if(fa->protocol < fb->protocol ) return(-1); else { if(fa->protocol > fb->protocol ) return(1); } - - return(0); -} - -/* ***************************************************** */ - -static struct ndpi_flow_info *get_ndpi_flow_info(u_int16_t thread_id, - const u_int8_t version, - u_int16_t vlan_id, - const struct ndpi_iphdr *iph, - const struct ndpi_ipv6hdr *iph6, - u_int16_t ip_offset, - u_int16_t ipsize, - u_int16_t l4_packet_len, - struct ndpi_tcphdr **tcph, - struct ndpi_udphdr **udph, - u_int16_t *sport, u_int16_t *dport, - struct ndpi_id_struct **src, - struct ndpi_id_struct **dst, - u_int8_t *proto, - u_int8_t **payload, - u_int16_t *payload_len, - u_int8_t *src_to_dst_direction) { - u_int32_t idx, l4_offset; - u_int32_t lower_ip; - u_int32_t upper_ip; - u_int16_t lower_port; - u_int16_t upper_port; - struct ndpi_flow_info flow; - void *ret; - u_int8_t *l3, *l4; - - /* - Note: to keep things simple (ndpiReader is just a demo app) - we handle IPv6 a-la-IPv4. - */ - if(version == 4) { - if(ipsize < 20) - return NULL; - - if((iph->ihl * 4) > ipsize || ipsize < ntohs(iph->tot_len) - || (iph->frag_off & htons(0x1FFF)) != 0) - return NULL; - - l4_offset = iph->ihl * 4; - l3 = (u_int8_t*)iph; - } else { - l4_offset = sizeof(struct ndpi_ipv6hdr); - l3 = (u_int8_t*)iph6; - } - - if(l4_packet_len < 64) - ndpi_thread_info[thread_id].stats.packet_len[0]++; - else if(l4_packet_len >= 64 && l4_packet_len < 128) - ndpi_thread_info[thread_id].stats.packet_len[1]++; - else if(l4_packet_len >= 128 && l4_packet_len < 256) - ndpi_thread_info[thread_id].stats.packet_len[2]++; - else if(l4_packet_len >= 256 && l4_packet_len < 1024) - ndpi_thread_info[thread_id].stats.packet_len[3]++; - else if(l4_packet_len >= 1024 && l4_packet_len < 1500) - ndpi_thread_info[thread_id].stats.packet_len[4]++; - else if(l4_packet_len >= 1500) - ndpi_thread_info[thread_id].stats.packet_len[5]++; - - if(l4_packet_len > ndpi_thread_info[thread_id].stats.max_packet_len) - ndpi_thread_info[thread_id].stats.max_packet_len = l4_packet_len; - - if(iph->saddr < iph->daddr) { - lower_ip = iph->saddr; - upper_ip = iph->daddr; - } else { - lower_ip = iph->daddr; - upper_ip = iph->saddr; - } - - *proto = iph->protocol; - l4 = ((u_int8_t *) l3 + l4_offset); - - if(iph->protocol == 6 && l4_packet_len >= 20) { - u_int tcp_len; - - ndpi_thread_info[thread_id].stats.tcp_count++; - - // tcp - *tcph = (struct ndpi_tcphdr *)l4; - *sport = ntohs((*tcph)->source), *dport = ntohs((*tcph)->dest); - - if(iph->saddr < iph->daddr) { - lower_port = (*tcph)->source, upper_port = (*tcph)->dest; - *src_to_dst_direction = 1; - } else { - lower_port = (*tcph)->dest; - upper_port = (*tcph)->source; - - *src_to_dst_direction = 0; - if(iph->saddr == iph->daddr) { - if(lower_port > upper_port) { - u_int16_t p = lower_port; - - lower_port = upper_port; - upper_port = p; - } - } - } - - tcp_len = ndpi_min(4*(*tcph)->doff, l4_packet_len); - *payload = &l4[tcp_len]; - *payload_len = ndpi_max(0, l4_packet_len-4*(*tcph)->doff); - } else if(iph->protocol == 17 && l4_packet_len >= 8) { - // udp - ndpi_thread_info[thread_id].stats.udp_count++; - - *udph = (struct ndpi_udphdr *)l4; - *sport = ntohs((*udph)->source), *dport = ntohs((*udph)->dest); - *payload = &l4[sizeof(struct ndpi_udphdr)]; - *payload_len = ndpi_max(0, l4_packet_len-sizeof(struct ndpi_udphdr)); - - if(iph->saddr < iph->daddr) { - lower_port = (*udph)->source, upper_port = (*udph)->dest; - *src_to_dst_direction = 1; - } else { - lower_port = (*udph)->dest, upper_port = (*udph)->source; - - *src_to_dst_direction = 0; - - if(iph->saddr == iph->daddr) { - if(lower_port > upper_port) { - u_int16_t p = lower_port; - - lower_port = upper_port; - upper_port = p; - } - } - } - - *sport = ntohs(lower_port), *dport = ntohs(upper_port); - } else { - // non tcp/udp protocols - lower_port = 0; - upper_port = 0; - } - - flow.protocol = iph->protocol, flow.vlan_id = vlan_id; - flow.lower_ip = lower_ip, flow.upper_ip = upper_ip; - flow.lower_port = lower_port, flow.upper_port = upper_port; - - if(0) - printf("[NDPI] [%u][%u:%u <-> %u:%u]\n", - iph->protocol, lower_ip, ntohs(lower_port), upper_ip, ntohs(upper_port)); - - idx = (vlan_id + lower_ip + upper_ip + iph->protocol + lower_port + upper_port) % NUM_ROOTS; - ret = ndpi_tfind(&flow, &ndpi_thread_info[thread_id].ndpi_flows_root[idx], node_cmp); - - if(ret == NULL) { - if(ndpi_thread_info[thread_id].stats.ndpi_flow_count == MAX_NDPI_FLOWS) { - printf("ERROR: maximum flow count (%u) has been exceeded\n", MAX_NDPI_FLOWS); - exit(-1); - } else { - struct ndpi_flow_info *newflow = (struct ndpi_flow_info*)malloc(sizeof(struct ndpi_flow_info)); - - if(newflow == NULL) { - printf("[NDPI] %s(1): not enough memory\n", __FUNCTION__); - return(NULL); - } - - memset(newflow, 0, sizeof(struct ndpi_flow_info)); - newflow->protocol = iph->protocol, newflow->vlan_id = vlan_id; - newflow->lower_ip = lower_ip, newflow->upper_ip = upper_ip; - newflow->lower_port = lower_port, newflow->upper_port = upper_port; - newflow->ip_version = version; - - if(version == 4) { - inet_ntop(AF_INET, &lower_ip, newflow->lower_name, sizeof(newflow->lower_name)); - inet_ntop(AF_INET, &upper_ip, newflow->upper_name, sizeof(newflow->upper_name)); - } else { - inet_ntop(AF_INET6, &iph6->ip6_src, newflow->lower_name, sizeof(newflow->lower_name)); - inet_ntop(AF_INET6, &iph6->ip6_dst, newflow->upper_name, sizeof(newflow->upper_name)); - } - - if((newflow->ndpi_flow = malloc_wrapper(size_flow_struct)) == NULL) { - printf("[NDPI] %s(2): not enough memory\n", __FUNCTION__); - free(newflow); - return(NULL); - } else - memset(newflow->ndpi_flow, 0, size_flow_struct); - - if((newflow->src_id = malloc_wrapper(size_id_struct)) == NULL) { - printf("[NDPI] %s(3): not enough memory\n", __FUNCTION__); - free(newflow); - return(NULL); - } else - memset(newflow->src_id, 0, size_id_struct); - - if((newflow->dst_id = malloc_wrapper(size_id_struct)) == NULL) { - printf("[NDPI] %s(4): not enough memory\n", __FUNCTION__); - free(newflow); - return(NULL); - } else - memset(newflow->dst_id, 0, size_id_struct); - - ndpi_tsearch(newflow, &ndpi_thread_info[thread_id].ndpi_flows_root[idx], node_cmp); /* Add */ - ndpi_thread_info[thread_id].stats.ndpi_flow_count++; - - *src = newflow->src_id, *dst = newflow->dst_id; - - // printFlow(thread_id, newflow); - return newflow; - } - } else { - struct ndpi_flow_info *flow = *(struct ndpi_flow_info**)ret; - - if(flow->lower_ip == lower_ip && flow->upper_ip == upper_ip - && flow->lower_port == lower_port && flow->upper_port == upper_port) - *src = flow->src_id, *dst = flow->dst_id; - else - *src = flow->dst_id, *dst = flow->src_id; - - return flow; - } -} - -/* ***************************************************** */ - -static struct ndpi_flow_info *get_ndpi_flow_info6(u_int16_t thread_id, - u_int16_t vlan_id, - const struct ndpi_ipv6hdr *iph6, - u_int16_t ip_offset, - struct ndpi_tcphdr **tcph, - struct ndpi_udphdr **udph, - u_int16_t *sport, u_int16_t *dport, - struct ndpi_id_struct **src, - struct ndpi_id_struct **dst, - u_int8_t *proto, - u_int8_t **payload, - u_int16_t *payload_len, - u_int8_t *src_to_dst_direction) { - struct ndpi_iphdr iph; - - memset(&iph, 0, sizeof(iph)); - iph.version = 4; - iph.saddr = iph6->ip6_src.u6_addr.u6_addr32[2] + iph6->ip6_src.u6_addr.u6_addr32[3]; - iph.daddr = iph6->ip6_dst.u6_addr.u6_addr32[2] + iph6->ip6_dst.u6_addr.u6_addr32[3]; - iph.protocol = iph6->ip6_ctlun.ip6_un1.ip6_un1_nxt; - - if(iph.protocol == 0x3C /* IPv6 destination option */) { - u_int8_t *options = (u_int8_t*)iph6 + sizeof(const struct ndpi_ipv6hdr); - - iph.protocol = options[0]; - } - - return(get_ndpi_flow_info(thread_id, 6, vlan_id, &iph, iph6, ip_offset, - sizeof(struct ndpi_ipv6hdr), - ntohs(iph6->ip6_ctlun.ip6_un1.ip6_un1_plen), - tcph, udph, sport, dport, - src, dst, proto, payload, payload_len, src_to_dst_direction)); -} - -/* ***************************************************** */ - -static void setupDetection(u_int16_t thread_id) { +static void setupDetection(u_int16_t thread_id, pcap_t * pcap_handle) { NDPI_PROTOCOL_BITMASK all; + + struct ndpi_workflow_prefs prefs; + memset(&prefs, 0, sizeof(prefs)); + prefs.decode_tunnels = decode_tunnels; + prefs.num_roots = NUM_ROOTS; + prefs.max_ndpi_flows = MAX_NDPI_FLOWS; + prefs.quiet_mode = quiet_mode; + prefs.detection_tick_resolution = detection_tick_resolution; memset(&ndpi_thread_info[thread_id], 0, sizeof(ndpi_thread_info[thread_id])); - - // init global detection structure - ndpi_thread_info[thread_id].ndpi_struct = ndpi_init_detection_module(detection_tick_resolution, - malloc_wrapper, free_wrapper, debug_printf); - if(ndpi_thread_info[thread_id].ndpi_struct == NULL) { - printf("ERROR: global structure initialization failed\n"); - exit(-1); - } - - /* ndpi_thread_info[thread_id].ndpi_struct->http_dont_dissect_response = 1; */ + ndpi_thread_info[thread_id].workflow = ndpi_workflow_init(&prefs, pcap_handle, malloc_wrapper, free_wrapper); + /* ndpi_thread_info[thread_id].workflow->ndpi_struct->http_dont_dissect_response = 1; */ // enable all protocols NDPI_BITMASK_SET_ALL(all); - ndpi_set_protocol_detection_bitmask2(ndpi_thread_info[thread_id].ndpi_struct, &all); - - // allocate memory for id and flow tracking - size_id_struct = sizeof(struct ndpi_id_struct); - size_flow_struct = sizeof(struct ndpi_flow_struct); + ndpi_set_protocol_detection_bitmask2(ndpi_thread_info[thread_id].workflow->ndpi_struct, &all); // clear memory for results - memset(ndpi_thread_info[thread_id].stats.protocol_counter, 0, sizeof(ndpi_thread_info[thread_id].stats.protocol_counter)); - memset(ndpi_thread_info[thread_id].stats.protocol_counter_bytes, 0, sizeof(ndpi_thread_info[thread_id].stats.protocol_counter_bytes)); - memset(ndpi_thread_info[thread_id].stats.protocol_flows, 0, sizeof(ndpi_thread_info[thread_id].stats.protocol_flows)); + memset(ndpi_thread_info[thread_id].workflow->stats.protocol_counter, 0, sizeof(ndpi_thread_info[thread_id].workflow->stats.protocol_counter)); + memset(ndpi_thread_info[thread_id].workflow->stats.protocol_counter_bytes, 0, sizeof(ndpi_thread_info[thread_id].workflow->stats.protocol_counter_bytes)); + memset(ndpi_thread_info[thread_id].workflow->stats.protocol_flows, 0, sizeof(ndpi_thread_info[thread_id].workflow->stats.protocol_flows)); if(_protoFilePath != NULL) - ndpi_load_protocols_file(ndpi_thread_info[thread_id].ndpi_struct, _protoFilePath); + ndpi_load_protocols_file(ndpi_thread_info[thread_id].workflow->ndpi_struct, _protoFilePath); } /* ***************************************************** */ @@ -1005,131 +611,13 @@ static void terminateDetection(u_int16_t thread_id) { int i; for(i=0; indpi_flows_root[i], ndpi_flow_info_freer); + ndpi_thread_info[thread_id].workflow->ndpi_flows_root[i] = NULL; } - ndpi_exit_detection_module(ndpi_thread_info[thread_id].ndpi_struct, free_wrapper); + ndpi_workflow_free(ndpi_thread_info[thread_id].workflow); } -/* ***************************************************** */ - -// ipsize = header->len - ip_offset ; rawsize = header->len -static unsigned int packet_processing(u_int16_t thread_id, - const u_int64_t time, - u_int16_t vlan_id, - const struct ndpi_iphdr *iph, - struct ndpi_ipv6hdr *iph6, - u_int16_t ip_offset, - u_int16_t ipsize, u_int16_t rawsize) { - struct ndpi_id_struct *src, *dst; - struct ndpi_flow_info *flow; - struct ndpi_flow_struct *ndpi_flow = NULL; - u_int8_t proto; - struct ndpi_tcphdr *tcph = NULL; - struct ndpi_udphdr *udph = NULL; - u_int16_t sport, dport, payload_len; - u_int8_t *payload; - u_int8_t src_to_dst_direction= 1; - - if(iph) - flow = get_ndpi_flow_info(thread_id, 4, vlan_id, iph, NULL, - ip_offset, ipsize, - ntohs(iph->tot_len) - (iph->ihl * 4), - &tcph, &udph, &sport, &dport, - &src, &dst, &proto, - &payload, &payload_len, &src_to_dst_direction); - else - flow = get_ndpi_flow_info6(thread_id, vlan_id, iph6, ip_offset, - &tcph, &udph, &sport, &dport, - &src, &dst, &proto, - &payload, &payload_len, &src_to_dst_direction); - - if(flow != NULL) { - ndpi_thread_info[thread_id].stats.ip_packet_count++; - ndpi_thread_info[thread_id].stats.total_wire_bytes += rawsize + 24 /* CRC etc */, ndpi_thread_info[thread_id].stats.total_ip_bytes += rawsize; - ndpi_flow = flow->ndpi_flow; - flow->packets++, flow->bytes += rawsize; - flow->last_seen = time; - } else { - return(0); - } - - if(flow->detection_completed) return(0); - - flow->detected_protocol = ndpi_detection_process_packet(ndpi_thread_info[thread_id].ndpi_struct, ndpi_flow, - iph ? (uint8_t *)iph : (uint8_t *)iph6, - ipsize, time, src, dst); - - if((flow->detected_protocol.protocol != NDPI_PROTOCOL_UNKNOWN) - || ((proto == IPPROTO_UDP) && (flow->packets > 8)) - || ((proto == IPPROTO_TCP) && (flow->packets > 10))) { - flow->detection_completed = 1; - - if((flow->detected_protocol.protocol == NDPI_PROTOCOL_UNKNOWN) && (ndpi_flow->num_stun_udp_pkts > 0)) - ndpi_set_detected_protocol(ndpi_thread_info[thread_id].ndpi_struct, ndpi_flow, NDPI_PROTOCOL_STUN, NDPI_PROTOCOL_UNKNOWN); - - snprintf(flow->host_server_name, sizeof(flow->host_server_name), "%s", flow->ndpi_flow->host_server_name); - - if(flow->detected_protocol.protocol == NDPI_PROTOCOL_BITTORRENT) { - int i, j, n = 0; - - for(i=0, j = 0; i<20; i++) { - sprintf(&flow->bittorent_hash[j], "%02x", flow->ndpi_flow->bittorent_hash[i]); - j += 2, n += flow->ndpi_flow->bittorent_hash[i]; - } - - if(n == 0) flow->bittorent_hash[0] = '\0'; - } - - if((proto == IPPROTO_TCP) && (flow->detected_protocol.protocol != NDPI_PROTOCOL_DNS)) { - snprintf(flow->ssl.client_certificate, sizeof(flow->ssl.client_certificate), "%s", flow->ndpi_flow->protos.ssl.client_certificate); - snprintf(flow->ssl.server_certificate, sizeof(flow->ssl.server_certificate), "%s", flow->ndpi_flow->protos.ssl.server_certificate); - } - - if(flow->detected_protocol.protocol == NDPI_PROTOCOL_UNKNOWN) - flow->detected_protocol = ndpi_detection_giveup(ndpi_thread_info[thread_id].ndpi_struct, flow->ndpi_flow); - - free_ndpi_flow_info(flow); - - if(verbose > 1) { - if(enable_protocol_guess) { - if(flow->detected_protocol.protocol == NDPI_PROTOCOL_UNKNOWN) { - flow->detected_protocol.protocol = node_guess_undetected_protocol(thread_id, flow), - flow->detected_protocol.master_protocol = NDPI_PROTOCOL_UNKNOWN; - } - } - - printFlow(thread_id, flow); - } - } - - if(live_capture) { - if(ndpi_thread_info[thread_id].last_idle_scan_time + IDLE_SCAN_PERIOD < ndpi_thread_info[thread_id].last_time) { - /* scan for idle flows */ - ndpi_twalk(ndpi_thread_info[thread_id].ndpi_flows_root[ndpi_thread_info[thread_id].idle_scan_idx], node_idle_scan_walker, &thread_id); - - /* remove idle flows (unfortunately we cannot do this inline) */ - while (ndpi_thread_info[thread_id].num_idle_flows > 0) { - - /* search and delete the idle flow from the "ndpi_flow_root" (see struct reader thread) - here flows are the node of a b-tree */ - ndpi_tdelete(ndpi_thread_info[thread_id].idle_flows[--ndpi_thread_info[thread_id].num_idle_flows], &ndpi_thread_info[thread_id].ndpi_flows_root[ndpi_thread_info[thread_id].idle_scan_idx], node_cmp); - - /* free the memory associated to idle flow in "idle_flows" - (see struct reader thread)*/ - free_ndpi_flow_info(ndpi_thread_info[thread_id].idle_flows[ndpi_thread_info[thread_id].num_idle_flows]); - ndpi_free(ndpi_thread_info[thread_id].idle_flows[ndpi_thread_info[thread_id].num_idle_flows]); - } - - if(++ndpi_thread_info[thread_id].idle_scan_idx == NUM_ROOTS) ndpi_thread_info[thread_id].idle_scan_idx = 0; - ndpi_thread_info[thread_id].last_idle_scan_time = ndpi_thread_info[thread_id].last_time; - } - } - - return 0; -} - -/* ****************************************************** */ - char* formatTraffic(float numBits, int bits, char *buf) { char unit; @@ -1215,7 +703,7 @@ static void printResults(u_int64_t tot_usec) { u_int32_t i; u_int64_t total_flow_bytes = 0; u_int avg_pkt_size = 0; - struct thread_stats cumulative_stats; + struct ndpi_stats cumulative_stats; int thread_id; char buf[32]; #ifdef HAVE_JSON_C @@ -1227,35 +715,35 @@ static void printResults(u_int64_t tot_usec) { memset(&cumulative_stats, 0, sizeof(cumulative_stats)); for(thread_id = 0; thread_id < num_threads; thread_id++) { - if(ndpi_thread_info[thread_id].stats.total_wire_bytes == 0) continue; + if(ndpi_thread_info[thread_id].workflow->stats.total_wire_bytes == 0) continue; for(i=0; indpi_flows_root[i], node_proto_guess_walker, &thread_id); /* Stats aggregation */ - cumulative_stats.guessed_flow_protocols += ndpi_thread_info[thread_id].stats.guessed_flow_protocols; - cumulative_stats.raw_packet_count += ndpi_thread_info[thread_id].stats.raw_packet_count; - cumulative_stats.ip_packet_count += ndpi_thread_info[thread_id].stats.ip_packet_count; - cumulative_stats.total_wire_bytes += ndpi_thread_info[thread_id].stats.total_wire_bytes; - cumulative_stats.total_ip_bytes += ndpi_thread_info[thread_id].stats.total_ip_bytes; - cumulative_stats.total_discarded_bytes += ndpi_thread_info[thread_id].stats.total_discarded_bytes; - - for(i = 0; i < ndpi_get_num_supported_protocols(ndpi_thread_info[0].ndpi_struct); i++) { - cumulative_stats.protocol_counter[i] += ndpi_thread_info[thread_id].stats.protocol_counter[i]; - cumulative_stats.protocol_counter_bytes[i] += ndpi_thread_info[thread_id].stats.protocol_counter_bytes[i]; - cumulative_stats.protocol_flows[i] += ndpi_thread_info[thread_id].stats.protocol_flows[i]; + cumulative_stats.guessed_flow_protocols += ndpi_thread_info[thread_id].workflow->stats.guessed_flow_protocols; + cumulative_stats.raw_packet_count += ndpi_thread_info[thread_id].workflow->stats.raw_packet_count; + cumulative_stats.ip_packet_count += ndpi_thread_info[thread_id].workflow->stats.ip_packet_count; + cumulative_stats.total_wire_bytes += ndpi_thread_info[thread_id].workflow->stats.total_wire_bytes; + cumulative_stats.total_ip_bytes += ndpi_thread_info[thread_id].workflow->stats.total_ip_bytes; + cumulative_stats.total_discarded_bytes += ndpi_thread_info[thread_id].workflow->stats.total_discarded_bytes; + + for(i = 0; i < ndpi_get_num_supported_protocols(ndpi_thread_info[0].workflow->ndpi_struct); i++) { + cumulative_stats.protocol_counter[i] += ndpi_thread_info[thread_id].workflow->stats.protocol_counter[i]; + cumulative_stats.protocol_counter_bytes[i] += ndpi_thread_info[thread_id].workflow->stats.protocol_counter_bytes[i]; + cumulative_stats.protocol_flows[i] += ndpi_thread_info[thread_id].workflow->stats.protocol_flows[i]; } - cumulative_stats.ndpi_flow_count += ndpi_thread_info[thread_id].stats.ndpi_flow_count; - cumulative_stats.tcp_count += ndpi_thread_info[thread_id].stats.tcp_count; - cumulative_stats.udp_count += ndpi_thread_info[thread_id].stats.udp_count; - cumulative_stats.mpls_count += ndpi_thread_info[thread_id].stats.mpls_count; - cumulative_stats.pppoe_count += ndpi_thread_info[thread_id].stats.pppoe_count; - cumulative_stats.vlan_count += ndpi_thread_info[thread_id].stats.vlan_count; - cumulative_stats.fragmented_count += ndpi_thread_info[thread_id].stats.fragmented_count; + cumulative_stats.ndpi_flow_count += ndpi_thread_info[thread_id].workflow->stats.ndpi_flow_count; + cumulative_stats.tcp_count += ndpi_thread_info[thread_id].workflow->stats.tcp_count; + cumulative_stats.udp_count += ndpi_thread_info[thread_id].workflow->stats.udp_count; + cumulative_stats.mpls_count += ndpi_thread_info[thread_id].workflow->stats.mpls_count; + cumulative_stats.pppoe_count += ndpi_thread_info[thread_id].workflow->stats.pppoe_count; + cumulative_stats.vlan_count += ndpi_thread_info[thread_id].workflow->stats.vlan_count; + cumulative_stats.fragmented_count += ndpi_thread_info[thread_id].workflow->stats.fragmented_count; for(i = 0; i < 6; i++) - cumulative_stats.packet_len[i] += ndpi_thread_info[thread_id].stats.packet_len[i]; - cumulative_stats.max_packet_len += ndpi_thread_info[thread_id].stats.max_packet_len; + cumulative_stats.packet_len[i] += ndpi_thread_info[thread_id].workflow->stats.packet_len[i]; + cumulative_stats.max_packet_len += ndpi_thread_info[thread_id].workflow->stats.max_packet_len; } if(!quiet_mode) { @@ -1352,15 +840,15 @@ static void printResults(u_int64_t tot_usec) { } if((!json_flag) && (!quiet_mode)) printf("\n\nDetected protocols:\n"); - for(i = 0; i <= ndpi_get_num_supported_protocols(ndpi_thread_info[0].ndpi_struct); i++) { - ndpi_protocol_breed_t breed = ndpi_get_proto_breed(ndpi_thread_info[0].ndpi_struct, i); + for(i = 0; i <= ndpi_get_num_supported_protocols(ndpi_thread_info[0].workflow->ndpi_struct); i++) { + ndpi_protocol_breed_t breed = ndpi_get_proto_breed(ndpi_thread_info[0].workflow->ndpi_struct, i); if(cumulative_stats.protocol_counter[i] > 0) { breed_stats[breed] += (long long unsigned int)cumulative_stats.protocol_counter_bytes[i]; if(results_file) fprintf(results_file, "%s\t%llu\t%llu\t%u\n", - ndpi_get_proto_name(ndpi_thread_info[0].ndpi_struct, i), + ndpi_get_proto_name(ndpi_thread_info[0].workflow->ndpi_struct, i), (long long unsigned int)cumulative_stats.protocol_counter[i], (long long unsigned int)cumulative_stats.protocol_counter_bytes[i], cumulative_stats.protocol_flows[i]); @@ -1368,7 +856,7 @@ static void printResults(u_int64_t tot_usec) { if((!json_flag) && (!quiet_mode)) { printf("\t%-20s packets: %-13llu bytes: %-13llu " "flows: %-13u\n", - ndpi_get_proto_name(ndpi_thread_info[0].ndpi_struct, i), + ndpi_get_proto_name(ndpi_thread_info[0].workflow->ndpi_struct, i), (long long unsigned int)cumulative_stats.protocol_counter[i], (long long unsigned int)cumulative_stats.protocol_counter_bytes[i], cumulative_stats.protocol_flows[i]); @@ -1377,8 +865,8 @@ static void printResults(u_int64_t tot_usec) { if(json_fp) { jObj = json_object_new_object(); - json_object_object_add(jObj,"name",json_object_new_string(ndpi_get_proto_name(ndpi_thread_info[0].ndpi_struct, i))); - json_object_object_add(jObj,"breed",json_object_new_string(ndpi_get_proto_breed_name(ndpi_thread_info[0].ndpi_struct, breed))); + json_object_object_add(jObj,"name",json_object_new_string(ndpi_get_proto_name(ndpi_thread_info[0].workflow->ndpi_struct, i))); + json_object_object_add(jObj,"breed",json_object_new_string(ndpi_get_proto_breed_name(ndpi_thread_info[0].workflow->ndpi_struct, breed))); json_object_object_add(jObj,"packets",json_object_new_int64(cumulative_stats.protocol_counter[i])); json_object_object_add(jObj,"bytes",json_object_new_int64(cumulative_stats.protocol_counter_bytes[i])); json_object_object_add(jObj,"flows",json_object_new_int(cumulative_stats.protocol_flows[i])); @@ -1398,7 +886,7 @@ static void printResults(u_int64_t tot_usec) { for(i=0; i < NUM_BREEDS; i++) { if(breed_stats[i] > 0) { printf("\t%-20s %13llu bytes\n", - ndpi_get_proto_breed_name(ndpi_thread_info[0].ndpi_struct, i), + ndpi_get_proto_breed_name(ndpi_thread_info[0].workflow->ndpi_struct, i), breed_stats[i]); } } @@ -1414,11 +902,11 @@ static void printResults(u_int64_t tot_usec) { num_flows = 0; for(thread_id = 0; thread_id < num_threads; thread_id++) { for(i=0; indpi_flows_root[i], node_print_known_proto_walker, &thread_id); } for(thread_id = 0; thread_id < num_threads; thread_id++) { - if(ndpi_thread_info[thread_id].stats.protocol_counter[0 /* 0 = Unknown */] > 0) { + if(ndpi_thread_info[thread_id].workflow->stats.protocol_counter[0 /* 0 = Unknown */] > 0) { if(!json_flag) { FILE *out = results_file ? results_file : stdout; @@ -1433,9 +921,9 @@ static void printResults(u_int64_t tot_usec) { num_flows = 0; for(thread_id = 0; thread_id < num_threads; thread_id++) { - if(ndpi_thread_info[thread_id].stats.protocol_counter[0] > 0) { + if(ndpi_thread_info[thread_id].workflow->stats.protocol_counter[0] > 0) { for(i=0; indpi_flows_root[i], node_print_unknown_proto_walker, &thread_id); } } } @@ -1456,18 +944,9 @@ static void printResults(u_int64_t tot_usec) { /* ***************************************************** */ -static void closePcapFile(u_int16_t thread_id) { - if(ndpi_thread_info[thread_id]._pcap_handle != NULL) { - pcap_close(ndpi_thread_info[thread_id]._pcap_handle); - - } -} - -/* ***************************************************** */ - static void breakPcapLoop(u_int16_t thread_id) { - if(ndpi_thread_info[thread_id]._pcap_handle != NULL) { - pcap_breakloop(ndpi_thread_info[thread_id]._pcap_handle); + if(ndpi_thread_info[thread_id].pcap_handle != NULL) { + pcap_breakloop(ndpi_thread_info[thread_id].pcap_handle); } } @@ -1509,17 +988,15 @@ static int getNextPcapFileFromPlaylist(u_int16_t thread_id, char filename[], u_i /* ***************************************************** */ -static void configurePcapHandle(u_int16_t thread_id) { - ndpi_thread_info[thread_id]._pcap_datalink_type = pcap_datalink(ndpi_thread_info[thread_id]._pcap_handle); - +static void configurePcapHandle(pcap_t * pcap_handle) { if(_bpf_filter != NULL) { struct bpf_program fcode; - if(pcap_compile(ndpi_thread_info[thread_id]._pcap_handle, &fcode, _bpf_filter, 1, 0xFFFFFF00) < 0) { - printf("pcap_compile error: '%s'\n", pcap_geterr(ndpi_thread_info[thread_id]._pcap_handle)); + if(pcap_compile(pcap_handle, &fcode, _bpf_filter, 1, 0xFFFFFF00) < 0) { + printf("pcap_compile error: '%s'\n", pcap_geterr(pcap_handle)); } else { - if(pcap_setfilter(ndpi_thread_info[thread_id]._pcap_handle, &fcode) < 0) { - printf("pcap_setfilter error: '%s'\n", pcap_geterr(ndpi_thread_info[thread_id]._pcap_handle)); + if(pcap_setfilter(pcap_handle, &fcode) < 0) { + printf("pcap_setfilter error: '%s'\n", pcap_geterr(pcap_handle)); } else printf("Successfully set BPF filter to '%s'\n", _bpf_filter); } @@ -1528,41 +1005,43 @@ static void configurePcapHandle(u_int16_t thread_id) { /* ***************************************************** */ -static void openPcapFileOrDevice(u_int16_t thread_id) { +/* Always returns a valid pcap_t */ +static pcap_t * openPcapFileOrDevice(u_int16_t thread_id, const u_char * pcap_file) { u_int snaplen = 1536; int promisc = 1; - char errbuf[PCAP_ERRBUF_SIZE]; + char pcap_error_buffer[PCAP_ERRBUF_SIZE]; + pcap_t * pcap_handle = NULL; /* trying to open a live interface */ - if((ndpi_thread_info[thread_id]._pcap_handle = pcap_open_live(_pcap_file[thread_id], snaplen, promisc, 500, errbuf)) == NULL) { + if((pcap_handle = pcap_open_live(pcap_file, snaplen, promisc, 500, pcap_error_buffer)) == NULL) { capture_for = capture_until = 0; live_capture = 0; num_threads = 1; /* Open pcap files in single threads mode */ /* trying to open a pcap file */ - if((ndpi_thread_info[thread_id]._pcap_handle = pcap_open_offline(_pcap_file[thread_id], ndpi_thread_info[thread_id]._pcap_error_buffer)) == NULL) { + if((pcap_handle = pcap_open_offline(pcap_file, pcap_error_buffer)) == NULL) { char filename[256]; /* trying to open a pcap playlist */ if(getNextPcapFileFromPlaylist(thread_id, filename, sizeof(filename)) != 0 || - (ndpi_thread_info[thread_id]._pcap_handle = pcap_open_offline(filename, ndpi_thread_info[thread_id]._pcap_error_buffer)) == NULL) { + (pcap_handle = pcap_open_offline(filename, pcap_error_buffer)) == NULL) { - printf("ERROR: could not open pcap file or playlist: %s\n", ndpi_thread_info[thread_id]._pcap_error_buffer); + printf("ERROR: could not open pcap file or playlist: %s\n", pcap_error_buffer); exit(-1); } else { - if((!json_flag) && (!quiet_mode)) printf("Reading packets from playlist %s...\n", _pcap_file[thread_id]); + if((!json_flag) && (!quiet_mode)) printf("Reading packets from playlist %s...\n", pcap_file); } } else { - if((!json_flag) && (!quiet_mode)) printf("Reading packets from pcap file %s...\n", _pcap_file[thread_id]); + if((!json_flag) && (!quiet_mode)) printf("Reading packets from pcap file %s...\n", pcap_file); } } else { live_capture = 1; - if((!json_flag) && (!quiet_mode)) printf("Capturing live traffic from device %s...\n", _pcap_file[thread_id]); + if((!json_flag) && (!quiet_mode)) printf("Capturing live traffic from device %s...\n", pcap_file); } - configurePcapHandle(thread_id); + configurePcapHandle(pcap_handle); if(capture_for > 0) { if((!json_flag) && (!quiet_mode)) printf("Capturing traffic up to %u seconds\n", (unsigned int)capture_for); @@ -1572,392 +1051,52 @@ static void openPcapFileOrDevice(u_int16_t thread_id) { signal(SIGALRM, sigproc); #endif } + + return pcap_handle; } /* ***************************************************** */ -static void pcap_packet_callback(u_char *args, +static void pcap_packet_callback_checked(u_char *args, const struct pcap_pkthdr *header, const u_char *packet) { - - /* - * Declare pointers to packet headers - */ - - /* --- Ethernet header --- */ - const struct ndpi_ethhdr *ethernet; - /* --- Ethernet II header --- */ - const struct ndpi_ethhdr *ethernet_2; - /* --- LLC header --- */ - const struct ndpi_llc_header *llc; - - /* --- Cisco HDLC header --- */ - const struct ndpi_chdlc *chdlc; - /* --- SLARP frame --- */ - struct ndpi_slarp *slarp; - /* --- CDP --- */ - struct ndpi_cdp *cdp; - - /* --- Radio Tap header --- */ - const struct ndpi_radiotap_header *radiotap; - /* --- Wifi header --- */ - const struct ndpi_wifi_header *wifi; - - /* --- MPLS header --- */ - struct ndpi_mpls_header *mpls; - - /** --- IP header --- **/ - struct ndpi_iphdr *iph; - /** --- IPv6 header --- **/ - struct ndpi_ipv6hdr *iph6; - - /* lengths and offsets */ - u_int16_t eth_offset = 0; - u_int16_t radio_len; - u_int16_t fc; - u_int16_t type; - int wifi_len; - int llc_off; - int pyld_eth_len = 0; - int check; - u_int32_t fcs; - u_int64_t time; - u_int16_t ip_offset, ip_len, ip6_offset; - u_int16_t frag_off = 0, vlan_id = 0; - u_int8_t proto = 0; - u_int32_t label; - u_int16_t thread_id = *((u_int16_t*)args); - - /* counters */ - u_int8_t malformed_pkts = 0, vlan_packet = 0; - u_int8_t slarp_pkts = 0, cdp_pkts = 0; - - /* Increment raw packet counter */ - ndpi_thread_info[thread_id].stats.raw_packet_count++; - + if((capture_until != 0) && (header->ts.tv_sec >= capture_until)) { - if(ndpi_thread_info[thread_id]._pcap_handle != NULL) - pcap_breakloop(ndpi_thread_info[thread_id]._pcap_handle); + if(ndpi_thread_info[thread_id].pcap_handle != NULL) + pcap_breakloop(ndpi_thread_info[thread_id].pcap_handle); return; } - + /* Check if capture is live or not */ if (!live_capture) { if (!pcap_start.tv_sec) pcap_start.tv_sec = header->ts.tv_sec, pcap_start.tv_usec = header->ts.tv_usec; pcap_end.tv_sec = header->ts.tv_sec, pcap_end.tv_usec = header->ts.tv_usec; } - - /* setting time */ - time = ((uint64_t) header->ts.tv_sec) * detection_tick_resolution + - header->ts.tv_usec / (1000000 / detection_tick_resolution); - - /* safety check */ - if(ndpi_thread_info[thread_id].last_time > time) { - /* printf("\nWARNING: timestamp bug in the pcap file (ts delta: %llu, repairing)\n", ndpi_thread_info[thread_id].last_time - time); */ - time = ndpi_thread_info[thread_id].last_time; - } - /* update last time value */ - ndpi_thread_info[thread_id].last_time = time; - - /*** check Data Link type ***/ - int datalink_type = ndpi_thread_info[thread_id]._pcap_datalink_type; - - datalink_check: - switch(datalink_type) { - case DLT_NULL : - if(ntohl(*((u_int32_t*)&packet[eth_offset])) == 2) - type = ETH_P_IP; - else - type = ETH_P_IPV6; - - ip_offset = 4 + eth_offset; - - /* Cisco PPP in HDLC-like framing - 50 */ - case DLT_PPP_SERIAL: - chdlc = (struct ndpi_chdlc *) &packet[eth_offset]; - ip_offset = sizeof(struct ndpi_chdlc); /* CHDLC_OFF = 4 */ - type = ntohs(chdlc->proto_code); - break; - - /* Cisco PPP with HDLC framing - 104 */ - case DLT_C_HDLC: - chdlc = (struct ndpi_chdlc *) &packet[eth_offset]; - ip_offset = sizeof(struct ndpi_chdlc); /* CHDLC_OFF = 4 */ - type = ntohs(chdlc->proto_code); - break; - - /* IEEE 802.3 Ethernet - 1 */ - case DLT_EN10MB : - ethernet = (struct ndpi_ethhdr *) &packet[eth_offset]; - ip_offset = sizeof(struct ndpi_ethhdr) + eth_offset; - check = ntohs(ethernet->h_proto); - - if(check <= 1500) - pyld_eth_len = check; - else if (check >= 1536) - type = check; - - if(pyld_eth_len != 0) { - /* check for LLC layer with SNAP extension */ - if(packet[ip_offset] == SNAP) { - llc = (struct ndpi_llc_header *)(&packet[ip_offset]); - type = llc->snap.proto_ID; - ip_offset += + 8; - } - } - break; - - /* Linux Cooked Capture - 113 */ - case DLT_LINUX_SLL : - type = (packet[eth_offset+14] << 8) + packet[eth_offset+15]; - ip_offset = 16 + eth_offset; - break; - - /* Radiotap link-layer - 127 */ - case DLT_IEEE802_11_RADIO : - radiotap = (struct ndpi_radiotap_header *) &packet[eth_offset]; - radio_len = radiotap->len; - - /* Check Bad FCS presence */ - if((radiotap->flags & BAD_FCS) == BAD_FCS) { - malformed_pkts += 1; - ndpi_thread_info[thread_id].stats.total_discarded_bytes += header->len; - return; - } - - fcs = header->len - 4; - - /* Calculate 802.11 header length (variable) */ - wifi = (struct ndpi_wifi_header*)( packet + eth_offset + radio_len); - fc = wifi->fc; - - /* check wifi data presence */ - if(FCF_TYPE(fc) == WIFI_DATA) { - if((FCF_TO_DS(fc) && FCF_FROM_DS(fc) == 0x0) || - (FCF_TO_DS(fc) == 0x0 && FCF_FROM_DS(fc))) - wifi_len = 26; /* + 4 byte fcs */ - } else /* no data frames */ - break; - - /* Check ether_type from LLC */ - llc = (struct ndpi_llc_header*)(packet + eth_offset + wifi_len + radio_len); - if(llc->dsap == SNAP) - type = ntohs(llc->snap.proto_ID); - - /* Set IP header offset */ - ip_offset = wifi_len + radio_len + sizeof(struct ndpi_llc_header) + eth_offset; - break; - - case DLT_RAW: - ip_offset = eth_offset = 0; - break; - - default: - /* printf("Unknown datalink %d\n", datalink_type); */ - return; - } - - /* check ether type */ - if(type == VLAN) { - vlan_id = ((packet[ip_offset] << 8) + packet[ip_offset+1]) & 0xFFF; - type = (packet[ip_offset+2] << 8) + packet[ip_offset+3]; - ip_offset += 4; - vlan_packet = 1; - } else if(type == MPLS_UNI || type == MPLS_MULTI) { - mpls = (struct ndpi_mpls_header *) &packet[ip_offset]; - label = ntohl(mpls->label); - /* label = ntohl(*((u_int32_t*)&packet[ip_offset])); */ - ndpi_thread_info[thread_id].stats.mpls_count++; - type = ETH_P_IP, ip_offset += 4; - - while((label & 0x100) != 0x100) { - ip_offset += 4; - label = ntohl(mpls->label); - } - } - else if(type == SLARP) { - slarp = (struct ndpi_slarp *) &packet[ip_offset]; - if(slarp->slarp_type == 0x02 || slarp->slarp_type == 0x00 || slarp->slarp_type == 0x01) { - /* TODO if info are needed */ - } - slarp_pkts++; - } - else if(type == CISCO_D_PROTO) { - cdp = (struct ndpi_cdp *) &packet[ip_offset]; - cdp_pkts++; - } - else if(type == PPPoE) { - ndpi_thread_info[thread_id].stats.pppoe_count++; - type = ETH_P_IP; - ip_offset += 8; - } - - ndpi_thread_info[thread_id].stats.vlan_count += vlan_packet; - - iph_check: - /* Check and set IP header size and total packet length */ - iph = (struct ndpi_iphdr *) &packet[ip_offset]; - - /* just work on Ethernet packets that contain IP */ - if(type == ETH_P_IP && header->caplen >= ip_offset) { - frag_off = ntohs(iph->frag_off); - - proto = iph->protocol; - if(header->caplen < header->len) { - static u_int8_t cap_warning_used = 0; - - if(cap_warning_used == 0) { - if((!json_flag) && (!quiet_mode)) printf("\n\nWARNING: packet capture size is smaller than packet size, DETECTION MIGHT NOT WORK CORRECTLY\n\n"); - cap_warning_used = 1; - } - } - } - - if(iph->version == 4) { - ip_len = ((u_short)iph->ihl * 4); - iph6 = NULL; - - if(iph->protocol == 41) { - ip_offset += ip_len; - goto iph_check; - } - - if((frag_off & 0x3FFF) != 0) { - static u_int8_t ipv4_frags_warning_used = 0; - ndpi_thread_info[thread_id].stats.fragmented_count++; - - if(ipv4_frags_warning_used == 0) { - if((!json_flag) && (!quiet_mode)) printf("\n\nWARNING: IPv4 fragments are not handled by this demo (nDPI supports them)\n"); - ipv4_frags_warning_used = 1; - } - - ndpi_thread_info[thread_id].stats.total_discarded_bytes += header->len; - return; - } - } else if(iph->version == 6) { - iph6 = (struct ndpi_ipv6hdr *)&packet[ip_offset]; - proto = iph6->ip6_ctlun.ip6_un1.ip6_un1_nxt; - ip_len = sizeof(struct ndpi_ipv6hdr); - - if(proto == 0x3C /* IPv6 destination option */) { - - u_int8_t *options = (u_int8_t*)&packet[ip_offset+ip_len]; - proto = options[0]; - ip_len += 8 * (options[1] + 1); - } - iph = NULL; - - } else { - static u_int8_t ipv4_warning_used = 0; - - v4_warning: - if(ipv4_warning_used == 0) { - if((!json_flag) && (!quiet_mode)) - printf("\n\nWARNING: only IPv4/IPv6 packets are supported in this demo (nDPI supports both IPv4 and IPv6), all other packets will be discarded\n\n"); - ipv4_warning_used = 1; - } - ndpi_thread_info[thread_id].stats.total_discarded_bytes += header->len; - return; - } - - if(decode_tunnels && (proto == IPPROTO_UDP)) { - struct ndpi_udphdr *udp = (struct ndpi_udphdr *)&packet[ip_offset+ip_len]; - u_int16_t sport = ntohs(udp->source), dport = ntohs(udp->dest); - - if((sport == GTP_U_V1_PORT) || (dport == GTP_U_V1_PORT)) { - /* Check if it's GTPv1 */ - u_int offset = ip_offset+ip_len+sizeof(struct ndpi_udphdr); - u_int8_t flags = packet[offset]; - u_int8_t message_type = packet[offset+1]; - - if((((flags & 0xE0) >> 5) == 1 /* GTPv1 */) && - (message_type == 0xFF /* T-PDU */)) { - - ip_offset = ip_offset+ip_len+sizeof(struct ndpi_udphdr)+8; /* GTPv1 header len */ - if(flags & 0x04) ip_offset += 1; /* next_ext_header is present */ - if(flags & 0x02) ip_offset += 4; /* sequence_number is present (it also includes next_ext_header and pdu_number) */ - if(flags & 0x01) ip_offset += 1; /* pdu_number is present */ - - iph = (struct ndpi_iphdr *) &packet[ip_offset]; - - if(iph->version != 4) { - // printf("WARNING: not good (packet_id=%u)!\n", (unsigned int)ndpi_thread_info[thread_id].stats.raw_packet_count); - goto v4_warning; - } - } - } else if((sport == TZSP_PORT) || (dport == TZSP_PORT)) { - /* https://en.wikipedia.org/wiki/TZSP */ - u_int offset = ip_offset+ip_len+sizeof(struct ndpi_udphdr); - u_int8_t version = packet[offset]; - u_int8_t type = packet[offset+1]; - u_int16_t encapsulates = ntohs(*((u_int16_t*)&packet[offset+2])); - - if((version == 1) && (type == 0) && (encapsulates == 1)) { - u_int8_t stop = 0; - - offset += 4; - - while((!stop) && (offset < header->caplen)) { - u_int8_t tag_type = packet[offset]; - u_int8_t tag_len; - - switch(tag_type) { - case 0: /* PADDING Tag */ - tag_len = 1; - break; - case 1: /* END Tag */ - tag_len = 1, stop = 1; - break; - default: - tag_len = packet[offset+1]; - break; - } - - offset += tag_len; - - if(offset >= header->caplen) - return; /* Invalid packet */ - else { - eth_offset = offset; - goto datalink_check; - } - } - } - } - } - - /* process the packet */ - packet_processing(thread_id, time, vlan_id, iph, iph6, - ip_offset, header->len - ip_offset, header->len); -} - -static void pcap_packet_callback_checked(u_char *args, - const struct pcap_pkthdr *header, - const u_char *packet) { - u_int16_t thread_id = *((u_int16_t*)args); + /* allocate an exact size buffer to check overflows */ uint8_t *packet_checked = malloc(header->caplen); memcpy(packet_checked, packet, header->caplen); - pcap_packet_callback(args, header, packet_checked); + ndpi_workflow_process_packet(ndpi_thread_info[thread_id].workflow, header, packet_checked); /* check for buffer changes */ if(memcmp(packet, packet_checked, header->caplen) != 0) printf("INTERNAL ERROR: ingress packet was nodified by nDPI: this should not happen [thread_id=%u, packetId=%lu]\n", - thread_id, (unsigned long)ndpi_thread_info[thread_id].stats.raw_packet_count); + thread_id, (unsigned long)ndpi_thread_info[thread_id].workflow->stats.raw_packet_count); free(packet_checked); } /* ******************************************************************** */ static void runPcapLoop(u_int16_t thread_id) { - if((!shutdown_app) && (ndpi_thread_info[thread_id]._pcap_handle != NULL)) - pcap_loop(ndpi_thread_info[thread_id]._pcap_handle, -1, &pcap_packet_callback_checked, (u_char*)&thread_id); + if((!shutdown_app) && (ndpi_thread_info[thread_id].pcap_handle != NULL)) + pcap_loop(ndpi_thread_info[thread_id].pcap_handle, -1, &pcap_packet_callback_checked, (u_char*)&thread_id); } /* ******************************************************************** */ void *processing_thread(void *_thread_id) { long thread_id = (long) _thread_id; + char pcap_error_buffer[PCAP_ERRBUF_SIZE]; #if defined(linux) && defined(HAVE_PTHREAD_SETAFFINITY_NP) if(core_affinity[thread_id] >= 0) { @@ -1981,8 +1120,8 @@ void *processing_thread(void *_thread_id) { char filename[256]; if(getNextPcapFileFromPlaylist(thread_id, filename, sizeof(filename)) == 0 && - (ndpi_thread_info[thread_id]._pcap_handle = pcap_open_offline(filename, ndpi_thread_info[thread_id]._pcap_error_buffer)) != NULL) { - configurePcapHandle(thread_id); + (ndpi_thread_info[thread_id].pcap_handle = pcap_open_offline(filename, pcap_error_buffer)) != NULL) { + configurePcapHandle(ndpi_thread_info[thread_id].pcap_handle); goto pcap_loop; } } @@ -2002,8 +1141,8 @@ void test_lib() { #endif for(thread_id = 0; thread_id < num_threads; thread_id++) { - setupDetection(thread_id); - openPcapFileOrDevice(thread_id); + pcap_t * cap = openPcapFileOrDevice(thread_id, _pcap_file[thread_id]); + setupDetection(thread_id, cap); } gettimeofday(&begin, NULL); @@ -2023,7 +1162,9 @@ void test_lib() { printResults(tot_usec); for(thread_id = 0; thread_id < num_threads; thread_id++) { - closePcapFile(thread_id); + if(ndpi_thread_info[thread_id].workflow->pcap_handle != NULL) { + pcap_close(ndpi_thread_info[thread_id].workflow->pcap_handle); + } terminateDetection(thread_id); } } diff --git a/src/include/ndpi_util.h b/src/include/ndpi_util.h new file mode 100644 index 000000000..33e84d050 --- /dev/null +++ b/src/include/ndpi_util.h @@ -0,0 +1,116 @@ +/* + * ndpi_util.h + * + * Copyright (C) 2011-15 - ntop.org + * Copyright (C) 2009-11 - ipoque GmbH + * + * This file is part of nDPI, an open source deep packet inspection + * library based on the OpenDPI and PACE technology by ipoque GmbH + * + * nDPI is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * nDPI is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with nDPI. If not, see . + * + */ + +/** + * This module contains routines to help setup a simple nDPI program. + * + * If you concern about performance or have to integrate nDPI in your + * application, you could need to reimplement them yourself. + * + * WARNING: this API is unstable! Use it at your own risk! + */ + +#include + +// flow tracking +typedef struct ndpi_flow_info { + u_int32_t lower_ip; + u_int32_t upper_ip; + u_int16_t lower_port; + u_int16_t upper_port; + u_int8_t detection_completed, protocol; + u_int16_t vlan_id; + struct ndpi_flow_struct *ndpi_flow; + char lower_name[48], upper_name[48]; + u_int8_t ip_version; + u_int64_t last_seen; + u_int64_t bytes; + u_int32_t packets; + + // result only, not used for flow identification + ndpi_protocol detected_protocol; + + char host_server_name[192]; + char bittorent_hash[41]; + + struct { + char client_certificate[48], server_certificate[48]; + } ssl; + + void *src_id, *dst_id; +} ndpi_flow_info_t; + +typedef struct ndpi_stats { + u_int32_t guessed_flow_protocols; + u_int64_t raw_packet_count; + u_int64_t ip_packet_count; + u_int64_t total_wire_bytes, total_ip_bytes, total_discarded_bytes; + u_int64_t protocol_counter[NDPI_MAX_SUPPORTED_PROTOCOLS + NDPI_MAX_NUM_CUSTOM_PROTOCOLS + 1]; + u_int64_t protocol_counter_bytes[NDPI_MAX_SUPPORTED_PROTOCOLS + NDPI_MAX_NUM_CUSTOM_PROTOCOLS + 1]; + u_int32_t protocol_flows[NDPI_MAX_SUPPORTED_PROTOCOLS + NDPI_MAX_NUM_CUSTOM_PROTOCOLS + 1]; + u_int32_t ndpi_flow_count; + u_int64_t tcp_count, udp_count; + u_int64_t mpls_count, pppoe_count, vlan_count, fragmented_count; + u_int64_t packet_len[6]; + u_int16_t max_packet_len; +} ndpi_stats_t; + +typedef struct ndpi_workflow_prefs { + u_int8_t decode_tunnels; + u_int8_t quiet_mode; + u_int32_t num_roots; + u_int32_t max_ndpi_flows; + u_int32_t detection_tick_resolution; +} ndpi_workflow_prefs_t; + +typedef struct ndpi_workflow { + u_int64_t last_time; + u_int64_t last_idle_scan_time; + u_int32_t idle_scan_idx; + u_int32_t num_idle_flows; /* TODO_EMA decide if idle flows will be handled */ + + struct ndpi_workflow_prefs prefs; + struct ndpi_stats stats; + + /* outside referencies */ + pcap_t *pcap_handle; + + /* allocated by prefs */ + struct ndpi_flow_info **idle_flows; + void **ndpi_flows_root; + struct ndpi_detection_module_struct *ndpi_struct; +} ndpi_workflow_t; + +/* TODO: remove wrappers parameters and use ndpi global, when their initialization will be fixed... */ +struct ndpi_workflow * ndpi_workflow_init(const struct ndpi_workflow_prefs * prefs, + pcap_t * pcap_handle, + void * (*malloc_wrapper)(size_t), + void (*free_wrapper)(void*)); + +void ndpi_workflow_free(struct ndpi_workflow * workflow); + +/** Process a @packet and update the @workflow. */ +void ndpi_workflow_process_packet (struct ndpi_workflow * workflow, + const struct pcap_pkthdr *header, + const u_char *packet); diff --git a/src/lib/ndpi_util.c b/src/lib/ndpi_util.c new file mode 100644 index 000000000..40a97f352 --- /dev/null +++ b/src/lib/ndpi_util.c @@ -0,0 +1,864 @@ + +#include + +#ifdef WIN32 +#include /* winsock.h is included automatically */ +#include +#include +#else +#include +#include +#endif + +#ifndef ETH_P_IP +#define ETH_P_IP 0x0800 /* IPv4 */ +#endif + +#ifndef ETH_P_IPv6 +#define ETH_P_IPV6 0x86dd /* IPv6 */ +#endif + +#define SLARP 0x8035 /* Cisco Slarp */ +#define CISCO_D_PROTO 0x2000 /* Cisco Discovery Protocol */ + +#define VLAN 0x8100 +#define MPLS_UNI 0x8847 +#define MPLS_MULTI 0x8848 +#define PPPoE 0x8864 +#define SNAP 0xaa + +/* mask for FCF */ +#define WIFI_DATA 0x2 /* 0000 0010 */ +#define FCF_TYPE(fc) (((fc) >> 2) & 0x3) /* 0000 0011 = 0x3 */ +#define FCF_SUBTYPE(fc) (((fc) >> 4) & 0xF) /* 0000 1111 = 0xF */ +#define FCF_TO_DS(fc) ((fc) & 0x0100) +#define FCF_FROM_DS(fc) ((fc) & 0x0200) + +/* mask for Bad FCF presence */ +#define BAD_FCS 0x50 /* 0101 0000 */ + +#define GTP_U_V1_PORT 2152 +#define TZSP_PORT 37008 + +#define SIZEOF_ID_STRUCT (sizeof(struct ndpi_id_struct)) +#define SIZEOF_FLOW_STRUCT (sizeof(struct ndpi_flow_struct)) + +#include "ndpi_main.h" +#include "ndpi_util.h" + +/* TODO_EMA printf replace with API */ +/* TODO_EMA */ +#define json_flag 0 + +/* ***************************************************** */ + +static void free_ndpi_flow_info(struct ndpi_flow_info *flow) { + if(flow->ndpi_flow) { ndpi_free_flow(flow->ndpi_flow); flow->ndpi_flow = NULL; } + if(flow->src_id) { ndpi_free(flow->src_id); flow->src_id = NULL; } + if(flow->dst_id) { ndpi_free(flow->dst_id); flow->dst_id = NULL; } + +} + +/* ***************************************************** */ + +static const u_int8_t nDPI_traceLevel = 0; + +static void debug_printf(u_int32_t protocol, void *id_struct, + ndpi_log_level_t log_level, + const char *format, ...) { + va_list va_ap; +#ifndef WIN32 + struct tm result; +#endif + + if(log_level <= nDPI_traceLevel) { + char buf[8192], out_buf[8192]; + char theDate[32]; + const char *extra_msg = ""; + time_t theTime = time(NULL); + + va_start (va_ap, format); + + if(log_level == NDPI_LOG_ERROR) + extra_msg = "ERROR: "; + else if(log_level == NDPI_LOG_TRACE) + extra_msg = "TRACE: "; + else + extra_msg = "DEBUG: "; + + memset(buf, 0, sizeof(buf)); + strftime(theDate, 32, "%d/%b/%Y %H:%M:%S", localtime_r(&theTime,&result) ); + vsnprintf(buf, sizeof(buf)-1, format, va_ap); + + snprintf(out_buf, sizeof(out_buf), "%s %s%s", theDate, extra_msg, buf); + printf("%s", out_buf); + fflush(stdout); + } + + va_end(va_ap); +} + +/* ***************************************************** */ +/* TODO remove in future... */ +static void (*removeme_free_wrapper)(void*); + +struct ndpi_workflow * ndpi_workflow_init(const struct ndpi_workflow_prefs * prefs, + pcap_t * pcap_handle, + void * (*malloc_wrapper)(size_t), + void (*free_wrapper)(void*)) { + + /* TODO: just needed here to init ndpi malloc wrapper */ + struct ndpi_detection_module_struct * module = ndpi_init_detection_module( + prefs->detection_tick_resolution, malloc_wrapper, free_wrapper, debug_printf); + + struct ndpi_workflow * workflow = ndpi_malloc(sizeof(struct ndpi_workflow)); + + removeme_free_wrapper = free_wrapper; + + workflow->pcap_handle = pcap_handle; + workflow->prefs = *prefs; + workflow->ndpi_struct = module; + + if(workflow->ndpi_struct == NULL) { + printf("ERROR: global structure initialization failed\n"); + exit(-1); + } + + workflow->idle_flows = ndpi_malloc(sizeof(struct ndpi_flow_info *) * workflow->num_idle_flows); + workflow->ndpi_flows_root = ndpi_malloc(sizeof(void *) * workflow->prefs.num_roots); +} + +void ndpi_workflow_free(struct ndpi_workflow * workflow) { + ndpi_exit_detection_module(workflow->ndpi_struct, removeme_free_wrapper); + free(workflow->ndpi_flows_root); + free(workflow->idle_flows); + free(workflow); +} + +/* ***************************************************** */ + +static int node_cmp(const void *a, const void *b) { + struct ndpi_flow_info *fa = (struct ndpi_flow_info*)a; + struct ndpi_flow_info *fb = (struct ndpi_flow_info*)b; + + if(fa->vlan_id < fb->vlan_id ) return(-1); else { if(fa->vlan_id > fb->vlan_id ) return(1); } + if(fa->lower_ip < fb->lower_ip ) return(-1); else { if(fa->lower_ip > fb->lower_ip ) return(1); } + if(fa->lower_port < fb->lower_port) return(-1); else { if(fa->lower_port > fb->lower_port) return(1); } + if(fa->upper_ip < fb->upper_ip ) return(-1); else { if(fa->upper_ip > fb->upper_ip ) return(1); } + if(fa->upper_port < fb->upper_port) return(-1); else { if(fa->upper_port > fb->upper_port) return(1); } + if(fa->protocol < fb->protocol ) return(-1); else { if(fa->protocol > fb->protocol ) return(1); } + + return(0); +} + +/* ***************************************************** */ + +static struct ndpi_flow_info *get_ndpi_flow_info(struct ndpi_workflow * workflow, + const u_int8_t version, + u_int16_t vlan_id, + const struct ndpi_iphdr *iph, + const struct ndpi_ipv6hdr *iph6, + u_int16_t ip_offset, + u_int16_t ipsize, + u_int16_t l4_packet_len, + struct ndpi_tcphdr **tcph, + struct ndpi_udphdr **udph, + u_int16_t *sport, u_int16_t *dport, + struct ndpi_id_struct **src, + struct ndpi_id_struct **dst, + u_int8_t *proto, + u_int8_t **payload, + u_int16_t *payload_len, + u_int8_t *src_to_dst_direction) { + u_int32_t idx, l4_offset; + u_int32_t lower_ip; + u_int32_t upper_ip; + u_int16_t lower_port; + u_int16_t upper_port; + struct ndpi_flow_info flow; + void *ret; + u_int8_t *l3, *l4; + + /* + Note: to keep things simple (ndpiReader is just a demo app) + we handle IPv6 a-la-IPv4. + */ + if(version == 4) { + if(ipsize < 20) + return NULL; + + if((iph->ihl * 4) > ipsize || ipsize < ntohs(iph->tot_len) + || (iph->frag_off & htons(0x1FFF)) != 0) + return NULL; + + l4_offset = iph->ihl * 4; + l3 = (u_int8_t*)iph; + } else { + l4_offset = sizeof(struct ndpi_ipv6hdr); + l3 = (u_int8_t*)iph6; + } + + if(l4_packet_len < 64) + workflow->stats.packet_len[0]++; + else if(l4_packet_len >= 64 && l4_packet_len < 128) + workflow->stats.packet_len[1]++; + else if(l4_packet_len >= 128 && l4_packet_len < 256) + workflow->stats.packet_len[2]++; + else if(l4_packet_len >= 256 && l4_packet_len < 1024) + workflow->stats.packet_len[3]++; + else if(l4_packet_len >= 1024 && l4_packet_len < 1500) + workflow->stats.packet_len[4]++; + else if(l4_packet_len >= 1500) + workflow->stats.packet_len[5]++; + + if(l4_packet_len > workflow->stats.max_packet_len) + workflow->stats.max_packet_len = l4_packet_len; + + if(iph->saddr < iph->daddr) { + lower_ip = iph->saddr; + upper_ip = iph->daddr; + } else { + lower_ip = iph->daddr; + upper_ip = iph->saddr; + } + + *proto = iph->protocol; + l4 = ((u_int8_t *) l3 + l4_offset); + + if(iph->protocol == 6 && l4_packet_len >= 20) { + u_int tcp_len; + + workflow->stats.tcp_count++; + + // tcp + *tcph = (struct ndpi_tcphdr *)l4; + *sport = ntohs((*tcph)->source), *dport = ntohs((*tcph)->dest); + + if(iph->saddr < iph->daddr) { + lower_port = (*tcph)->source, upper_port = (*tcph)->dest; + *src_to_dst_direction = 1; + } else { + lower_port = (*tcph)->dest; + upper_port = (*tcph)->source; + + *src_to_dst_direction = 0; + if(iph->saddr == iph->daddr) { + if(lower_port > upper_port) { + u_int16_t p = lower_port; + + lower_port = upper_port; + upper_port = p; + } + } + } + + tcp_len = ndpi_min(4*(*tcph)->doff, l4_packet_len); + *payload = &l4[tcp_len]; + *payload_len = ndpi_max(0, l4_packet_len-4*(*tcph)->doff); + } else if(iph->protocol == 17 && l4_packet_len >= 8) { + // udp + workflow->stats.udp_count++; + + *udph = (struct ndpi_udphdr *)l4; + *sport = ntohs((*udph)->source), *dport = ntohs((*udph)->dest); + *payload = &l4[sizeof(struct ndpi_udphdr)]; + *payload_len = ndpi_max(0, l4_packet_len-sizeof(struct ndpi_udphdr)); + + if(iph->saddr < iph->daddr) { + lower_port = (*udph)->source, upper_port = (*udph)->dest; + *src_to_dst_direction = 1; + } else { + lower_port = (*udph)->dest, upper_port = (*udph)->source; + + *src_to_dst_direction = 0; + + if(iph->saddr == iph->daddr) { + if(lower_port > upper_port) { + u_int16_t p = lower_port; + + lower_port = upper_port; + upper_port = p; + } + } + } + + *sport = ntohs(lower_port), *dport = ntohs(upper_port); + } else { + // non tcp/udp protocols + lower_port = 0; + upper_port = 0; + } + + flow.protocol = iph->protocol, flow.vlan_id = vlan_id; + flow.lower_ip = lower_ip, flow.upper_ip = upper_ip; + flow.lower_port = lower_port, flow.upper_port = upper_port; + + if(0) + printf("[NDPI] [%u][%u:%u <-> %u:%u]\n", + iph->protocol, lower_ip, ntohs(lower_port), upper_ip, ntohs(upper_port)); + + idx = (vlan_id + lower_ip + upper_ip + iph->protocol + lower_port + upper_port) % workflow->prefs.num_roots; + ret = ndpi_tfind(&flow, &workflow->ndpi_flows_root[idx], node_cmp); + + if(ret == NULL) { + if(workflow->stats.ndpi_flow_count == workflow->prefs.max_ndpi_flows) { + printf("ERROR: maximum flow count (%u) has been exceeded\n", workflow->prefs.max_ndpi_flows); + exit(-1); + } else { + struct ndpi_flow_info *newflow = (struct ndpi_flow_info*)malloc(sizeof(struct ndpi_flow_info)); + + if(newflow == NULL) { + printf("[NDPI] %s(1): not enough memory\n", __FUNCTION__); + return(NULL); + } + + memset(newflow, 0, sizeof(struct ndpi_flow_info)); + newflow->protocol = iph->protocol, newflow->vlan_id = vlan_id; + newflow->lower_ip = lower_ip, newflow->upper_ip = upper_ip; + newflow->lower_port = lower_port, newflow->upper_port = upper_port; + newflow->ip_version = version; + + if(version == 4) { + inet_ntop(AF_INET, &lower_ip, newflow->lower_name, sizeof(newflow->lower_name)); + inet_ntop(AF_INET, &upper_ip, newflow->upper_name, sizeof(newflow->upper_name)); + } else { + inet_ntop(AF_INET6, &iph6->ip6_src, newflow->lower_name, sizeof(newflow->lower_name)); + inet_ntop(AF_INET6, &iph6->ip6_dst, newflow->upper_name, sizeof(newflow->upper_name)); + } + + if((newflow->ndpi_flow = ndpi_malloc(SIZEOF_FLOW_STRUCT)) == NULL) { + printf("[NDPI] %s(2): not enough memory\n", __FUNCTION__); + free(newflow); + return(NULL); + } else + memset(newflow->ndpi_flow, 0, SIZEOF_FLOW_STRUCT); + + if((newflow->src_id = ndpi_malloc(SIZEOF_ID_STRUCT)) == NULL) { + printf("[NDPI] %s(3): not enough memory\n", __FUNCTION__); + free(newflow); + return(NULL); + } else + memset(newflow->src_id, 0, SIZEOF_ID_STRUCT); + + if((newflow->dst_id = ndpi_malloc(SIZEOF_ID_STRUCT)) == NULL) { + printf("[NDPI] %s(4): not enough memory\n", __FUNCTION__); + free(newflow); + return(NULL); + } else + memset(newflow->dst_id, 0, SIZEOF_ID_STRUCT); + + ndpi_tsearch(newflow, &workflow->ndpi_flows_root[idx], node_cmp); /* Add */ + workflow->stats.ndpi_flow_count++; + + *src = newflow->src_id, *dst = newflow->dst_id; + + return newflow; + } + } else { + struct ndpi_flow_info *flow = *(struct ndpi_flow_info**)ret; + + if(flow->lower_ip == lower_ip && flow->upper_ip == upper_ip + && flow->lower_port == lower_port && flow->upper_port == upper_port) + *src = flow->src_id, *dst = flow->dst_id; + else + *src = flow->dst_id, *dst = flow->src_id; + + return flow; + } +} + +/* ****************************************************** */ + +static struct ndpi_flow_info *get_ndpi_flow_info6(struct ndpi_workflow * workflow, + u_int16_t vlan_id, + const struct ndpi_ipv6hdr *iph6, + u_int16_t ip_offset, + struct ndpi_tcphdr **tcph, + struct ndpi_udphdr **udph, + u_int16_t *sport, u_int16_t *dport, + struct ndpi_id_struct **src, + struct ndpi_id_struct **dst, + u_int8_t *proto, + u_int8_t **payload, + u_int16_t *payload_len, + u_int8_t *src_to_dst_direction) { + struct ndpi_iphdr iph; + + memset(&iph, 0, sizeof(iph)); + iph.version = 4; + iph.saddr = iph6->ip6_src.u6_addr.u6_addr32[2] + iph6->ip6_src.u6_addr.u6_addr32[3]; + iph.daddr = iph6->ip6_dst.u6_addr.u6_addr32[2] + iph6->ip6_dst.u6_addr.u6_addr32[3]; + iph.protocol = iph6->ip6_ctlun.ip6_un1.ip6_un1_nxt; + + if(iph.protocol == 0x3C /* IPv6 destination option */) { + u_int8_t *options = (u_int8_t*)iph6 + sizeof(const struct ndpi_ipv6hdr); + + iph.protocol = options[0]; + } + + return(get_ndpi_flow_info(workflow, 6, vlan_id, &iph, iph6, ip_offset, + sizeof(struct ndpi_ipv6hdr), + ntohs(iph6->ip6_ctlun.ip6_un1.ip6_un1_plen), + tcph, udph, sport, dport, + src, dst, proto, payload, payload_len, src_to_dst_direction)); +} + +/* ****************************************************** */ + +// ipsize = header->len - ip_offset ; rawsize = header->len +static unsigned int packet_processing(struct ndpi_workflow * workflow, + const u_int64_t time, + u_int16_t vlan_id, + const struct ndpi_iphdr *iph, + struct ndpi_ipv6hdr *iph6, + u_int16_t ip_offset, + u_int16_t ipsize, u_int16_t rawsize) { + struct ndpi_id_struct *src, *dst; + struct ndpi_flow_info *flow; + struct ndpi_flow_struct *ndpi_flow = NULL; + u_int8_t proto; + struct ndpi_tcphdr *tcph = NULL; + struct ndpi_udphdr *udph = NULL; + u_int16_t sport, dport, payload_len; + u_int8_t *payload; + u_int8_t src_to_dst_direction= 1; + + if(iph) + flow = get_ndpi_flow_info(workflow, 4, vlan_id, iph, NULL, + ip_offset, ipsize, + ntohs(iph->tot_len) - (iph->ihl * 4), + &tcph, &udph, &sport, &dport, + &src, &dst, &proto, + &payload, &payload_len, &src_to_dst_direction); + else + flow = get_ndpi_flow_info6(workflow, vlan_id, iph6, ip_offset, + &tcph, &udph, &sport, &dport, + &src, &dst, &proto, + &payload, &payload_len, &src_to_dst_direction); + + if(flow != NULL) { + workflow->stats.ip_packet_count++; + workflow->stats.total_wire_bytes += rawsize + 24 /* CRC etc */, workflow->stats.total_ip_bytes += rawsize; + ndpi_flow = flow->ndpi_flow; + flow->packets++, flow->bytes += rawsize; + flow->last_seen = time; + } else { + return(0); + } + + if(flow->detection_completed) return(0); + + flow->detected_protocol = ndpi_detection_process_packet(workflow->ndpi_struct, ndpi_flow, + iph ? (uint8_t *)iph : (uint8_t *)iph6, + ipsize, time, src, dst); + + if((flow->detected_protocol.protocol != NDPI_PROTOCOL_UNKNOWN) + || ((proto == IPPROTO_UDP) && (flow->packets > 8)) + || ((proto == IPPROTO_TCP) && (flow->packets > 10))) { + flow->detection_completed = 1; + + if((flow->detected_protocol.protocol == NDPI_PROTOCOL_UNKNOWN) && (ndpi_flow->num_stun_udp_pkts > 0)) + ndpi_set_detected_protocol(workflow->ndpi_struct, ndpi_flow, NDPI_PROTOCOL_STUN, NDPI_PROTOCOL_UNKNOWN); + + snprintf(flow->host_server_name, sizeof(flow->host_server_name), "%s", flow->ndpi_flow->host_server_name); + + if(flow->detected_protocol.protocol == NDPI_PROTOCOL_BITTORRENT) { + int i, j, n = 0; + + for(i=0, j = 0; i<20; i++) { + sprintf(&flow->bittorent_hash[j], "%02x", flow->ndpi_flow->bittorent_hash[i]); + j += 2, n += flow->ndpi_flow->bittorent_hash[i]; + } + + if(n == 0) flow->bittorent_hash[0] = '\0'; + } + + if((proto == IPPROTO_TCP) && (flow->detected_protocol.protocol != NDPI_PROTOCOL_DNS)) { + snprintf(flow->ssl.client_certificate, sizeof(flow->ssl.client_certificate), "%s", flow->ndpi_flow->protos.ssl.client_certificate); + snprintf(flow->ssl.server_certificate, sizeof(flow->ssl.server_certificate), "%s", flow->ndpi_flow->protos.ssl.server_certificate); + } + + if(flow->detected_protocol.protocol == NDPI_PROTOCOL_UNKNOWN) + flow->detected_protocol = ndpi_detection_giveup(workflow->ndpi_struct, flow->ndpi_flow); + + free_ndpi_flow_info(flow); + + /* TODO_EMA */ + /*if(verbose > 1) { + if(enable_protocol_guess) { + if(flow->detected_protocol.protocol == NDPI_PROTOCOL_UNKNOWN) { + flow->detected_protocol.protocol = node_guess_undetected_protocol(thread_id, flow), + flow->detected_protocol.master_protocol = NDPI_PROTOCOL_UNKNOWN; + } + } + + printFlow(thread_id, flow); + }*/ + } + + /* TODO_EMA use a callback */ + //~ if(live_capture) { + //~ if(ndpi_thread_info[thread_id].last_idle_scan_time + IDLE_SCAN_PERIOD < ndpi_thread_info[thread_id].last_time) { + //~ /* scan for idle flows */ + //~ ndpi_twalk(workflow->ndpi_flows_root[ndpi_thread_info[thread_id].idle_scan_idx], node_idle_scan_walker, &thread_id); + + //~ /* remove idle flows (unfortunately we cannot do this inline) */ + //~ while (ndpi_thread_info[thread_id].num_idle_flows > 0) { + + //~ /* search and delete the idle flow from the "ndpi_flow_root" (see struct reader thread) - here flows are the node of a b-tree */ + //~ ndpi_tdelete(ndpi_thread_info[thread_id].idle_flows[--ndpi_thread_info[thread_id].num_idle_flows], &workflow->ndpi_flows_root[ndpi_thread_info[thread_id].idle_scan_idx], node_cmp); + + //~ /* free the memory associated to idle flow in "idle_flows" - (see struct reader thread)*/ + //~ free_ndpi_flow_info(ndpi_thread_info[thread_id].idle_flows[ndpi_thread_info[thread_id].num_idle_flows]); + //~ ndpi_free(ndpi_thread_info[thread_id].idle_flows[ndpi_thread_info[thread_id].num_idle_flows]); + //~ } + + //~ if(++ndpi_thread_info[thread_id].idle_scan_idx == workflow->prefs.num_roots) ndpi_thread_info[thread_id].idle_scan_idx = 0; + //~ ndpi_thread_info[thread_id].last_idle_scan_time = ndpi_thread_info[thread_id].last_time; + //~ } + //~ } + + return 0; +} + +/* ****************************************************** */ +void ndpi_workflow_process_packet (struct ndpi_workflow * workflow, + const struct pcap_pkthdr *header, + const u_char *packet) { + /* + * Declare pointers to packet headers + */ + + /* --- Ethernet header --- */ + const struct ndpi_ethhdr *ethernet; + /* --- Ethernet II header --- */ + const struct ndpi_ethhdr *ethernet_2; + /* --- LLC header --- */ + const struct ndpi_llc_header *llc; + + /* --- Cisco HDLC header --- */ + const struct ndpi_chdlc *chdlc; + /* --- SLARP frame --- */ + struct ndpi_slarp *slarp; + /* --- CDP --- */ + struct ndpi_cdp *cdp; + + /* --- Radio Tap header --- */ + const struct ndpi_radiotap_header *radiotap; + /* --- Wifi header --- */ + const struct ndpi_wifi_header *wifi; + + /* --- MPLS header --- */ + struct ndpi_mpls_header *mpls; + + /** --- IP header --- **/ + struct ndpi_iphdr *iph; + /** --- IPv6 header --- **/ + struct ndpi_ipv6hdr *iph6; + + /* lengths and offsets */ + u_int16_t eth_offset = 0; + u_int16_t radio_len; + u_int16_t fc; + u_int16_t type; + int wifi_len; + int llc_off; + int pyld_eth_len = 0; + int check; + u_int32_t fcs; + u_int64_t time; + u_int16_t ip_offset, ip_len, ip6_offset; + u_int16_t frag_off = 0, vlan_id = 0; + u_int8_t proto = 0; + u_int32_t label; + + /* counters */ + u_int8_t malformed_pkts = 0, vlan_packet = 0; + u_int8_t slarp_pkts = 0, cdp_pkts = 0; + + /* Increment raw packet counter */ + workflow->stats.raw_packet_count++; + + /* setting time */ + time = ((uint64_t) header->ts.tv_sec) * workflow->prefs.detection_tick_resolution + + header->ts.tv_usec / (1000000 / workflow->prefs.detection_tick_resolution); + + /* safety check */ + if(workflow->last_time > time) { + /* printf("\nWARNING: timestamp bug in the pcap file (ts delta: %llu, repairing)\n", ndpi_thread_info[thread_id].last_time - time); */ + time = workflow->last_time; + } + /* update last time value */ + workflow->last_time = time; + + /*** check Data Link type ***/ + const int datalink_type = pcap_datalink(workflow->pcap_handle); + + datalink_check: + switch(datalink_type) { + case DLT_NULL : + if(ntohl(*((u_int32_t*)&packet[eth_offset])) == 2) + type = ETH_P_IP; + else + type = ETH_P_IPV6; + + ip_offset = 4 + eth_offset; + + /* Cisco PPP in HDLC-like framing - 50 */ + case DLT_PPP_SERIAL: + chdlc = (struct ndpi_chdlc *) &packet[eth_offset]; + ip_offset = sizeof(struct ndpi_chdlc); /* CHDLC_OFF = 4 */ + type = ntohs(chdlc->proto_code); + break; + + /* Cisco PPP with HDLC framing - 104 */ + case DLT_C_HDLC: + chdlc = (struct ndpi_chdlc *) &packet[eth_offset]; + ip_offset = sizeof(struct ndpi_chdlc); /* CHDLC_OFF = 4 */ + type = ntohs(chdlc->proto_code); + break; + + /* IEEE 802.3 Ethernet - 1 */ + case DLT_EN10MB : + ethernet = (struct ndpi_ethhdr *) &packet[eth_offset]; + ip_offset = sizeof(struct ndpi_ethhdr) + eth_offset; + check = ntohs(ethernet->h_proto); + + if(check <= 1500) + pyld_eth_len = check; + else if (check >= 1536) + type = check; + + if(pyld_eth_len != 0) { + /* check for LLC layer with SNAP extension */ + if(packet[ip_offset] == SNAP) { + llc = (struct ndpi_llc_header *)(&packet[ip_offset]); + type = llc->snap.proto_ID; + ip_offset += + 8; + } + } + break; + + /* Linux Cooked Capture - 113 */ + case DLT_LINUX_SLL : + type = (packet[eth_offset+14] << 8) + packet[eth_offset+15]; + ip_offset = 16 + eth_offset; + break; + + /* Radiotap link-layer - 127 */ + case DLT_IEEE802_11_RADIO : + radiotap = (struct ndpi_radiotap_header *) &packet[eth_offset]; + radio_len = radiotap->len; + + /* Check Bad FCS presence */ + if((radiotap->flags & BAD_FCS) == BAD_FCS) { + malformed_pkts += 1; + workflow->stats.total_discarded_bytes += header->len; + return; + } + + fcs = header->len - 4; + + /* Calculate 802.11 header length (variable) */ + wifi = (struct ndpi_wifi_header*)( packet + eth_offset + radio_len); + fc = wifi->fc; + + /* check wifi data presence */ + if(FCF_TYPE(fc) == WIFI_DATA) { + if((FCF_TO_DS(fc) && FCF_FROM_DS(fc) == 0x0) || + (FCF_TO_DS(fc) == 0x0 && FCF_FROM_DS(fc))) + wifi_len = 26; /* + 4 byte fcs */ + } else /* no data frames */ + break; + + /* Check ether_type from LLC */ + llc = (struct ndpi_llc_header*)(packet + eth_offset + wifi_len + radio_len); + if(llc->dsap == SNAP) + type = ntohs(llc->snap.proto_ID); + + /* Set IP header offset */ + ip_offset = wifi_len + radio_len + sizeof(struct ndpi_llc_header) + eth_offset; + break; + + case DLT_RAW: + ip_offset = eth_offset = 0; + break; + + default: + /* printf("Unknown datalink %d\n", datalink_type); */ + return; + } + + /* check ether type */ + if(type == VLAN) { + vlan_id = ((packet[ip_offset] << 8) + packet[ip_offset+1]) & 0xFFF; + type = (packet[ip_offset+2] << 8) + packet[ip_offset+3]; + ip_offset += 4; + vlan_packet = 1; + } else if(type == MPLS_UNI || type == MPLS_MULTI) { + mpls = (struct ndpi_mpls_header *) &packet[ip_offset]; + label = ntohl(mpls->label); + /* label = ntohl(*((u_int32_t*)&packet[ip_offset])); */ + workflow->stats.mpls_count++; + type = ETH_P_IP, ip_offset += 4; + + while((label & 0x100) != 0x100) { + ip_offset += 4; + label = ntohl(mpls->label); + } + } + else if(type == SLARP) { + slarp = (struct ndpi_slarp *) &packet[ip_offset]; + if(slarp->slarp_type == 0x02 || slarp->slarp_type == 0x00 || slarp->slarp_type == 0x01) { + /* TODO if info are needed */ + } + slarp_pkts++; + } + else if(type == CISCO_D_PROTO) { + cdp = (struct ndpi_cdp *) &packet[ip_offset]; + cdp_pkts++; + } + else if(type == PPPoE) { + workflow->stats.pppoe_count++; + type = ETH_P_IP; + ip_offset += 8; + } + + workflow->stats.vlan_count += vlan_packet; + + iph_check: + /* Check and set IP header size and total packet length */ + iph = (struct ndpi_iphdr *) &packet[ip_offset]; + + /* just work on Ethernet packets that contain IP */ + if(type == ETH_P_IP && header->caplen >= ip_offset) { + frag_off = ntohs(iph->frag_off); + + proto = iph->protocol; + if(header->caplen < header->len) { + static u_int8_t cap_warning_used = 0; + + if(cap_warning_used == 0) { + if((!json_flag) && (!workflow->prefs.quiet_mode)) printf("\n\nWARNING: packet capture size is smaller than packet size, DETECTION MIGHT NOT WORK CORRECTLY\n\n"); + cap_warning_used = 1; + } + } + } + + if(iph->version == 4) { + ip_len = ((u_short)iph->ihl * 4); + iph6 = NULL; + + if(iph->protocol == 41) { + ip_offset += ip_len; + goto iph_check; + } + + if((frag_off & 0x3FFF) != 0) { + static u_int8_t ipv4_frags_warning_used = 0; + workflow->stats.fragmented_count++; + + if(ipv4_frags_warning_used == 0) { + if((!json_flag) && (!workflow->prefs.quiet_mode)) printf("\n\nWARNING: IPv4 fragments are not handled by this demo (nDPI supports them)\n"); + ipv4_frags_warning_used = 1; + } + + workflow->stats.total_discarded_bytes += header->len; + return; + } + } else if(iph->version == 6) { + iph6 = (struct ndpi_ipv6hdr *)&packet[ip_offset]; + proto = iph6->ip6_ctlun.ip6_un1.ip6_un1_nxt; + ip_len = sizeof(struct ndpi_ipv6hdr); + + if(proto == 0x3C /* IPv6 destination option */) { + + u_int8_t *options = (u_int8_t*)&packet[ip_offset+ip_len]; + proto = options[0]; + ip_len += 8 * (options[1] + 1); + } + iph = NULL; + + } else { + static u_int8_t ipv4_warning_used = 0; + + v4_warning: + if(ipv4_warning_used == 0) { + if((!json_flag) && (!workflow->prefs.quiet_mode)) + printf("\n\nWARNING: only IPv4/IPv6 packets are supported in this demo (nDPI supports both IPv4 and IPv6), all other packets will be discarded\n\n"); + ipv4_warning_used = 1; + } + workflow->stats.total_discarded_bytes += header->len; + return; + } + + if(workflow->prefs.decode_tunnels && (proto == IPPROTO_UDP)) { + struct ndpi_udphdr *udp = (struct ndpi_udphdr *)&packet[ip_offset+ip_len]; + u_int16_t sport = ntohs(udp->source), dport = ntohs(udp->dest); + + if((sport == GTP_U_V1_PORT) || (dport == GTP_U_V1_PORT)) { + /* Check if it's GTPv1 */ + u_int offset = ip_offset+ip_len+sizeof(struct ndpi_udphdr); + u_int8_t flags = packet[offset]; + u_int8_t message_type = packet[offset+1]; + + if((((flags & 0xE0) >> 5) == 1 /* GTPv1 */) && + (message_type == 0xFF /* T-PDU */)) { + + ip_offset = ip_offset+ip_len+sizeof(struct ndpi_udphdr)+8; /* GTPv1 header len */ + if(flags & 0x04) ip_offset += 1; /* next_ext_header is present */ + if(flags & 0x02) ip_offset += 4; /* sequence_number is present (it also includes next_ext_header and pdu_number) */ + if(flags & 0x01) ip_offset += 1; /* pdu_number is present */ + + iph = (struct ndpi_iphdr *) &packet[ip_offset]; + + if(iph->version != 4) { + // printf("WARNING: not good (packet_id=%u)!\n", (unsigned int)workflow->stats.raw_packet_count); + goto v4_warning; + } + } + } else if((sport == TZSP_PORT) || (dport == TZSP_PORT)) { + /* https://en.wikipedia.org/wiki/TZSP */ + u_int offset = ip_offset+ip_len+sizeof(struct ndpi_udphdr); + u_int8_t version = packet[offset]; + u_int8_t type = packet[offset+1]; + u_int16_t encapsulates = ntohs(*((u_int16_t*)&packet[offset+2])); + + if((version == 1) && (type == 0) && (encapsulates == 1)) { + u_int8_t stop = 0; + + offset += 4; + + while((!stop) && (offset < header->caplen)) { + u_int8_t tag_type = packet[offset]; + u_int8_t tag_len; + + switch(tag_type) { + case 0: /* PADDING Tag */ + tag_len = 1; + break; + case 1: /* END Tag */ + tag_len = 1, stop = 1; + break; + default: + tag_len = packet[offset+1]; + break; + } + + offset += tag_len; + + if(offset >= header->caplen) + return; /* Invalid packet */ + else { + eth_offset = offset; + goto datalink_check; + } + } + } + } + } + + /* process the packet */ + packet_processing(workflow, time, vlan_id, iph, iph6, + ip_offset, header->len - ip_offset, header->len); +} -- cgit v1.2.3 From d7548e72b265198528279bd1ad87b04cdd6221c6 Mon Sep 17 00:00:00 2001 From: emanuele-f Date: Tue, 19 Apr 2016 23:35:35 +0200 Subject: Fix some issues --- example/ndpiReader.c | 30 +++++++++++++++--------------- src/lib/Makefile.am | 1 + src/lib/ndpi_util.c | 1 + 3 files changed, 17 insertions(+), 15 deletions(-) (limited to 'example/ndpiReader.c') diff --git a/example/ndpiReader.c b/example/ndpiReader.c index c2cfcf33d..d86fb3f2b 100644 --- a/example/ndpiReader.c +++ b/example/ndpiReader.c @@ -109,7 +109,6 @@ static u_int32_t num_flows; struct reader_thread { struct ndpi_workflow * workflow; pthread_t pthread; - pcap_t * pcap_handle; }; static struct reader_thread ndpi_thread_info[MAX_NUM_READER_THREADS]; @@ -945,8 +944,8 @@ static void printResults(u_int64_t tot_usec) { /* ***************************************************** */ static void breakPcapLoop(u_int16_t thread_id) { - if(ndpi_thread_info[thread_id].pcap_handle != NULL) { - pcap_breakloop(ndpi_thread_info[thread_id].pcap_handle); + if(ndpi_thread_info[thread_id].workflow->pcap_handle != NULL) { + pcap_breakloop(ndpi_thread_info[thread_id].workflow->pcap_handle); } } @@ -1061,10 +1060,15 @@ static void pcap_packet_callback_checked(u_char *args, const struct pcap_pkthdr *header, const u_char *packet) { u_int16_t thread_id = *((u_int16_t*)args); - + + /* allocate an exact size buffer to check overflows */ + uint8_t *packet_checked = malloc(header->caplen); + memcpy(packet_checked, packet, header->caplen); + ndpi_workflow_process_packet(ndpi_thread_info[thread_id].workflow, header, packet_checked); + if((capture_until != 0) && (header->ts.tv_sec >= capture_until)) { - if(ndpi_thread_info[thread_id].pcap_handle != NULL) - pcap_breakloop(ndpi_thread_info[thread_id].pcap_handle); + if(ndpi_thread_info[thread_id].workflow->pcap_handle != NULL) + pcap_breakloop(ndpi_thread_info[thread_id].workflow->pcap_handle); return; } @@ -1073,11 +1077,7 @@ static void pcap_packet_callback_checked(u_char *args, if (!pcap_start.tv_sec) pcap_start.tv_sec = header->ts.tv_sec, pcap_start.tv_usec = header->ts.tv_usec; pcap_end.tv_sec = header->ts.tv_sec, pcap_end.tv_usec = header->ts.tv_usec; } - - /* allocate an exact size buffer to check overflows */ - uint8_t *packet_checked = malloc(header->caplen); - memcpy(packet_checked, packet, header->caplen); - ndpi_workflow_process_packet(ndpi_thread_info[thread_id].workflow, header, packet_checked); + /* check for buffer changes */ if(memcmp(packet, packet_checked, header->caplen) != 0) printf("INTERNAL ERROR: ingress packet was nodified by nDPI: this should not happen [thread_id=%u, packetId=%lu]\n", @@ -1088,8 +1088,8 @@ static void pcap_packet_callback_checked(u_char *args, /* ******************************************************************** */ static void runPcapLoop(u_int16_t thread_id) { - if((!shutdown_app) && (ndpi_thread_info[thread_id].pcap_handle != NULL)) - pcap_loop(ndpi_thread_info[thread_id].pcap_handle, -1, &pcap_packet_callback_checked, (u_char*)&thread_id); + if((!shutdown_app) && (ndpi_thread_info[thread_id].workflow->pcap_handle != NULL)) + pcap_loop(ndpi_thread_info[thread_id].workflow->pcap_handle, -1, &pcap_packet_callback_checked, (u_char*)&thread_id); } /* ******************************************************************** */ @@ -1120,8 +1120,8 @@ void *processing_thread(void *_thread_id) { char filename[256]; if(getNextPcapFileFromPlaylist(thread_id, filename, sizeof(filename)) == 0 && - (ndpi_thread_info[thread_id].pcap_handle = pcap_open_offline(filename, pcap_error_buffer)) != NULL) { - configurePcapHandle(ndpi_thread_info[thread_id].pcap_handle); + (ndpi_thread_info[thread_id].workflow->pcap_handle = pcap_open_offline(filename, pcap_error_buffer)) != NULL) { + configurePcapHandle(ndpi_thread_info[thread_id].workflow->pcap_handle); goto pcap_loop; } } diff --git a/src/lib/Makefile.am b/src/lib/Makefile.am index d83fdd5c0..312cdc88c 100644 --- a/src/lib/Makefile.am +++ b/src/lib/Makefile.am @@ -16,6 +16,7 @@ libndpi_la_include_HEADERS = ../include/ndpi_api.h \ libndpi_la_SOURCES = ndpi_content_match.c.inc \ ndpi_main.c \ + ndpi_util.c \ protocols/afp.c \ protocols/aimini.c \ protocols/applejuice.c \ diff --git a/src/lib/ndpi_util.c b/src/lib/ndpi_util.c index 1bc75aad0..3028fd5b6 100644 --- a/src/lib/ndpi_util.c +++ b/src/lib/ndpi_util.c @@ -149,6 +149,7 @@ struct ndpi_workflow * ndpi_workflow_init(const struct ndpi_workflow_prefs * pre workflow->idle_flows = ndpi_malloc(sizeof(struct ndpi_flow_info *) * workflow->num_idle_flows); workflow->ndpi_flows_root = ndpi_malloc(sizeof(void *) * workflow->prefs.num_roots); + return workflow; } void ndpi_workflow_free(struct ndpi_workflow * workflow) { -- cgit v1.2.3 From 9ee64420045028f20a655bce34fe795aefdb288e Mon Sep 17 00:00:00 2001 From: emanuele-f Date: Wed, 20 Apr 2016 01:32:01 +0200 Subject: Other util API stuff --- example/ndpiReader.c | 92 ++++++++++++++++++++++++++++++++++-- src/include/ndpi_util.h | 33 +++++++++++-- src/lib/ndpi_util.c | 123 ++++++++++++------------------------------------ 3 files changed, 147 insertions(+), 101 deletions(-) (limited to 'example/ndpiReader.c') diff --git a/example/ndpiReader.c b/example/ndpiReader.c index d86fb3f2b..0a1747088 100644 --- a/example/ndpiReader.c +++ b/example/ndpiReader.c @@ -109,6 +109,10 @@ static u_int32_t num_flows; struct reader_thread { struct ndpi_workflow * workflow; pthread_t pthread; + u_int64_t last_idle_scan_time; + u_int32_t idle_scan_idx; + u_int32_t num_idle_flows; + struct ndpi_flow_info *idle_flows[IDLE_SCAN_BUDGET]; }; static struct reader_thread ndpi_thread_info[MAX_NUM_READER_THREADS]; @@ -553,7 +557,7 @@ static void node_idle_scan_walker(const void *node, ndpi_VISIT which, int depth, struct ndpi_flow_info *flow = *(struct ndpi_flow_info **) node; u_int16_t thread_id = *((u_int16_t *) user_data); - if(ndpi_thread_info[thread_id].workflow->num_idle_flows == IDLE_SCAN_BUDGET) /* TODO optimise with a budget-based walk */ + if(ndpi_thread_info[thread_id].num_idle_flows == IDLE_SCAN_BUDGET) /* TODO optimise with a budget-based walk */ return; if((which == ndpi_preorder) || (which == ndpi_leaf)) { /* Avoid walking the same node multiple times */ @@ -569,13 +573,69 @@ static void node_idle_scan_walker(const void *node, ndpi_VISIT which, int depth, ndpi_thread_info[thread_id].workflow->stats.ndpi_flow_count--; /* adding to a queue (we can't delete it from the tree inline ) */ - ndpi_thread_info[thread_id].workflow->idle_flows[ndpi_thread_info[thread_id].workflow->num_idle_flows++] = flow; + ndpi_thread_info[thread_id].idle_flows[ndpi_thread_info[thread_id].num_idle_flows++] = flow; } } } /* ***************************************************** */ +static void on_protocol_discovered(struct ndpi_workflow * workflow, + struct ndpi_flow_info * flow, + void * udata) { + const u_int16_t thread_id = *((u_int16_t *)udata); + + if(verbose > 1) { + if(enable_protocol_guess) { + if(flow->detected_protocol.protocol == NDPI_PROTOCOL_UNKNOWN) { + flow->detected_protocol.protocol = node_guess_undetected_protocol(thread_id, flow), + flow->detected_protocol.master_protocol = NDPI_PROTOCOL_UNKNOWN; + } + } + + printFlow(thread_id, flow); + } +} + +/* ***************************************************** */ + +static void debug_printf(u_int32_t protocol, void *id_struct, + ndpi_log_level_t log_level, + const char *format, ...) { + va_list va_ap; +#ifndef WIN32 + struct tm result; +#endif + + if(log_level <= nDPI_traceLevel) { + char buf[8192], out_buf[8192]; + char theDate[32]; + const char *extra_msg = ""; + time_t theTime = time(NULL); + + va_start (va_ap, format); + + if(log_level == NDPI_LOG_ERROR) + extra_msg = "ERROR: "; + else if(log_level == NDPI_LOG_TRACE) + extra_msg = "TRACE: "; + else + extra_msg = "DEBUG: "; + + memset(buf, 0, sizeof(buf)); + strftime(theDate, 32, "%d/%b/%Y %H:%M:%S", localtime_r(&theTime,&result) ); + vsnprintf(buf, sizeof(buf)-1, format, va_ap); + + snprintf(out_buf, sizeof(out_buf), "%s %s%s", theDate, extra_msg, buf); + printf("%s", out_buf); + fflush(stdout); + } + + va_end(va_ap); +} + +/* ***************************************************** */ + static void setupDetection(u_int16_t thread_id, pcap_t * pcap_handle) { NDPI_PROTOCOL_BITMASK all; @@ -588,8 +648,10 @@ static void setupDetection(u_int16_t thread_id, pcap_t * pcap_handle) { prefs.detection_tick_resolution = detection_tick_resolution; memset(&ndpi_thread_info[thread_id], 0, sizeof(ndpi_thread_info[thread_id])); - ndpi_thread_info[thread_id].workflow = ndpi_workflow_init(&prefs, pcap_handle, malloc_wrapper, free_wrapper); + ndpi_thread_info[thread_id].workflow = ndpi_workflow_init(&prefs, pcap_handle, malloc_wrapper, free_wrapper, debug_printf); /* ndpi_thread_info[thread_id].workflow->ndpi_struct->http_dont_dissect_response = 1; */ + + ndpi_workflow_set_flow_detected_callback(ndpi_thread_info[thread_id].workflow, on_protocol_discovered, (void *)&thread_id); // enable all protocols NDPI_BITMASK_SET_ALL(all); @@ -1078,6 +1140,30 @@ static void pcap_packet_callback_checked(u_char *args, pcap_end.tv_sec = header->ts.tv_sec, pcap_end.tv_usec = header->ts.tv_usec; } + /* Idle flows cleanup */ + if(live_capture) { + if(ndpi_thread_info[thread_id].last_idle_scan_time + IDLE_SCAN_PERIOD < ndpi_thread_info[thread_id].workflow->last_time) { + /* scan for idle flows */ + ndpi_twalk(ndpi_thread_info[thread_id].workflow->ndpi_flows_root[ndpi_thread_info[thread_id].idle_scan_idx], node_idle_scan_walker, &thread_id); + + /* remove idle flows (unfortunately we cannot do this inline) */ + while (ndpi_thread_info[thread_id].num_idle_flows > 0) { + + /* search and delete the idle flow from the "ndpi_flow_root" (see struct reader thread) - here flows are the node of a b-tree */ + ndpi_tdelete(ndpi_thread_info[thread_id].idle_flows[--ndpi_thread_info[thread_id].num_idle_flows], + &ndpi_thread_info[thread_id].workflow->ndpi_flows_root[ndpi_thread_info[thread_id].idle_scan_idx], + ndpi_workflow_node_cmp); + + /* free the memory associated to idle flow in "idle_flows" - (see struct reader thread)*/ + free_ndpi_flow_info(ndpi_thread_info[thread_id].idle_flows[ndpi_thread_info[thread_id].num_idle_flows]); + ndpi_free(ndpi_thread_info[thread_id].idle_flows[ndpi_thread_info[thread_id].num_idle_flows]); + } + + if(++ndpi_thread_info[thread_id].idle_scan_idx == ndpi_thread_info[thread_id].workflow->prefs.num_roots) ndpi_thread_info[thread_id].idle_scan_idx = 0; + ndpi_thread_info[thread_id].last_idle_scan_time = ndpi_thread_info[thread_id].workflow->last_time; + } + } + /* check for buffer changes */ if(memcmp(packet, packet_checked, header->caplen) != 0) printf("INTERNAL ERROR: ingress packet was nodified by nDPI: this should not happen [thread_id=%u, packetId=%lu]\n", diff --git a/src/include/ndpi_util.h b/src/include/ndpi_util.h index c7d1e6745..daa564b69 100644 --- a/src/include/ndpi_util.h +++ b/src/include/ndpi_util.h @@ -87,20 +87,25 @@ typedef struct ndpi_workflow_prefs { u_int32_t detection_tick_resolution; } ndpi_workflow_prefs_t; +struct ndpi_workflow; +/** workflow, flow, user data */ +typedef void (*ndpi_workflow_callback_ptr) (struct ndpi_workflow *, struct ndpi_flow_info *, void *); + typedef struct ndpi_workflow { u_int64_t last_time; - u_int64_t last_idle_scan_time; - u_int32_t idle_scan_idx; - u_int32_t num_idle_flows; /* TODO_EMA decide if idle flows will be handled */ struct ndpi_workflow_prefs prefs; struct ndpi_stats stats; + ndpi_workflow_callback_ptr __flow_detected_callback; + void * __flow_detected_udata; + ndpi_workflow_callback_ptr __flow_giveup_callback; + void * __flow_giveup_udata; + /* outside referencies */ pcap_t *pcap_handle; /* allocated by prefs */ - struct ndpi_flow_info **idle_flows; void **ndpi_flows_root; struct ndpi_detection_module_struct *ndpi_struct; } ndpi_workflow_t; @@ -109,7 +114,8 @@ typedef struct ndpi_workflow { struct ndpi_workflow * ndpi_workflow_init(const struct ndpi_workflow_prefs * prefs, pcap_t * pcap_handle, void * (*malloc_wrapper)(size_t), - void (*free_wrapper)(void*)); + void (*free_wrapper)(void*), + ndpi_debug_function_ptr ndpi_debug_printf); void ndpi_workflow_free(struct ndpi_workflow * workflow); @@ -117,5 +123,22 @@ void ndpi_workflow_free(struct ndpi_workflow * workflow); void ndpi_workflow_process_packet (struct ndpi_workflow * workflow, const struct pcap_pkthdr *header, const u_char *packet); + +/* flow callbacks: ndpi_flow_info will be freed right after */ +inline void ndpi_workflow_set_flow_detected_callback(struct ndpi_workflow * workflow, + ndpi_workflow_callback_ptr callback, + void * udata) { + workflow->__flow_detected_callback = callback; + workflow->__flow_detected_udata = udata; +} + +inline void ndpi_workflow_set_flow_giveup_callback(struct ndpi_workflow * workflow, + ndpi_workflow_callback_ptr callback, + void * udata) { + workflow->__flow_giveup_callback = callback; + workflow->__flow_giveup_udata = udata; +} + +int ndpi_workflow_node_cmp(const void *a, const void *b); #endif diff --git a/src/lib/ndpi_util.c b/src/lib/ndpi_util.c index 5899c81a1..51d6757e1 100644 --- a/src/lib/ndpi_util.c +++ b/src/lib/ndpi_util.c @@ -69,10 +69,6 @@ #include "ndpi_main.h" #include "ndpi_util.h" -/* TODO_EMA printf replace with API */ -/* TODO_EMA */ -#define json_flag 0 - /* ***************************************************** */ static void free_ndpi_flow_info(struct ndpi_flow_info *flow) { @@ -86,41 +82,6 @@ static void free_ndpi_flow_info(struct ndpi_flow_info *flow) { static const u_int8_t nDPI_traceLevel = 0; -static void debug_printf(u_int32_t protocol, void *id_struct, - ndpi_log_level_t log_level, - const char *format, ...) { - va_list va_ap; -#ifndef WIN32 - struct tm result; -#endif - - if(log_level <= nDPI_traceLevel) { - char buf[8192], out_buf[8192]; - char theDate[32]; - const char *extra_msg = ""; - time_t theTime = time(NULL); - - va_start (va_ap, format); - - if(log_level == NDPI_LOG_ERROR) - extra_msg = "ERROR: "; - else if(log_level == NDPI_LOG_TRACE) - extra_msg = "TRACE: "; - else - extra_msg = "DEBUG: "; - - memset(buf, 0, sizeof(buf)); - strftime(theDate, 32, "%d/%b/%Y %H:%M:%S", localtime_r(&theTime,&result) ); - vsnprintf(buf, sizeof(buf)-1, format, va_ap); - - snprintf(out_buf, sizeof(out_buf), "%s %s%s", theDate, extra_msg, buf); - printf("%s", out_buf); - fflush(stdout); - } - - va_end(va_ap); -} - /* ***************************************************** */ /* TODO remove in future... */ static void (*removeme_free_wrapper)(void*); @@ -128,11 +89,12 @@ static void (*removeme_free_wrapper)(void*); struct ndpi_workflow * ndpi_workflow_init(const struct ndpi_workflow_prefs * prefs, pcap_t * pcap_handle, void * (*malloc_wrapper)(size_t), - void (*free_wrapper)(void*)) { + void (*free_wrapper)(void*), + ndpi_debug_function_ptr ndpi_debug_printf) { /* TODO: just needed here to init ndpi malloc wrapper */ struct ndpi_detection_module_struct * module = ndpi_init_detection_module( - prefs->detection_tick_resolution, malloc_wrapper, free_wrapper, debug_printf); + prefs->detection_tick_resolution, malloc_wrapper, free_wrapper, ndpi_debug_printf); struct ndpi_workflow * workflow = ndpi_calloc(1, sizeof(struct ndpi_workflow)); @@ -143,11 +105,10 @@ struct ndpi_workflow * ndpi_workflow_init(const struct ndpi_workflow_prefs * pre workflow->ndpi_struct = module; if(workflow->ndpi_struct == NULL) { - printf("ERROR: global structure initialization failed\n"); + NDPI_LOG(0, NULL, NDPI_LOG_ERROR, "global structure initialization failed\n"); exit(-1); } - workflow->idle_flows = ndpi_calloc(workflow->num_idle_flows, sizeof(struct ndpi_flow_info *)); workflow->ndpi_flows_root = ndpi_calloc(workflow->prefs.num_roots, sizeof(void *)); return workflow; } @@ -155,13 +116,12 @@ struct ndpi_workflow * ndpi_workflow_init(const struct ndpi_workflow_prefs * pre void ndpi_workflow_free(struct ndpi_workflow * workflow) { ndpi_exit_detection_module(workflow->ndpi_struct, removeme_free_wrapper); free(workflow->ndpi_flows_root); - free(workflow->idle_flows); free(workflow); } /* ***************************************************** */ -static int node_cmp(const void *a, const void *b) { +int ndpi_workflow_node_cmp(const void *a, const void *b) { struct ndpi_flow_info *fa = (struct ndpi_flow_info*)a; struct ndpi_flow_info *fb = (struct ndpi_flow_info*)b; @@ -318,21 +278,21 @@ static struct ndpi_flow_info *get_ndpi_flow_info(struct ndpi_workflow * workflow flow.lower_port = lower_port, flow.upper_port = upper_port; if(0) - printf("[NDPI] [%u][%u:%u <-> %u:%u]\n", + NDPI_LOG(0, workflow.ndpi_struct, NDPI_LOG_DEBUG, "[NDPI] [%u][%u:%u <-> %u:%u]\n", iph->protocol, lower_ip, ntohs(lower_port), upper_ip, ntohs(upper_port)); idx = (vlan_id + lower_ip + upper_ip + iph->protocol + lower_port + upper_port) % workflow->prefs.num_roots; - ret = ndpi_tfind(&flow, &workflow->ndpi_flows_root[idx], node_cmp); + ret = ndpi_tfind(&flow, &workflow->ndpi_flows_root[idx], ndpi_workflow_node_cmp); if(ret == NULL) { if(workflow->stats.ndpi_flow_count == workflow->prefs.max_ndpi_flows) { - printf("ERROR: maximum flow count (%u) has been exceeded\n", workflow->prefs.max_ndpi_flows); + NDPI_LOG(0, workflow.ndpi_struct, NDPI_LOG_ERROR, "maximum flow count (%u) has been exceeded\n", workflow->prefs.max_ndpi_flows); exit(-1); } else { struct ndpi_flow_info *newflow = (struct ndpi_flow_info*)malloc(sizeof(struct ndpi_flow_info)); if(newflow == NULL) { - printf("[NDPI] %s(1): not enough memory\n", __FUNCTION__); + NDPI_LOG(0, workflow.ndpi_struct, NDPI_LOG_ERROR, "[NDPI] %s(1): not enough memory\n", __FUNCTION__); return(NULL); } @@ -351,27 +311,27 @@ static struct ndpi_flow_info *get_ndpi_flow_info(struct ndpi_workflow * workflow } if((newflow->ndpi_flow = ndpi_malloc(SIZEOF_FLOW_STRUCT)) == NULL) { - printf("[NDPI] %s(2): not enough memory\n", __FUNCTION__); + NDPI_LOG(0, workflow.ndpi_struct, NDPI_LOG_ERROR, "[NDPI] %s(2): not enough memory\n", __FUNCTION__); free(newflow); return(NULL); } else memset(newflow->ndpi_flow, 0, SIZEOF_FLOW_STRUCT); if((newflow->src_id = ndpi_malloc(SIZEOF_ID_STRUCT)) == NULL) { - printf("[NDPI] %s(3): not enough memory\n", __FUNCTION__); + NDPI_LOG(0, workflow.ndpi_struct, NDPI_LOG_ERROR, "[NDPI] %s(3): not enough memory\n", __FUNCTION__); free(newflow); return(NULL); } else memset(newflow->src_id, 0, SIZEOF_ID_STRUCT); if((newflow->dst_id = ndpi_malloc(SIZEOF_ID_STRUCT)) == NULL) { - printf("[NDPI] %s(4): not enough memory\n", __FUNCTION__); + NDPI_LOG(0, workflow.ndpi_struct, NDPI_LOG_ERROR, "[NDPI] %s(4): not enough memory\n", __FUNCTION__); free(newflow); return(NULL); } else memset(newflow->dst_id, 0, SIZEOF_ID_STRUCT); - ndpi_tsearch(newflow, &workflow->ndpi_flows_root[idx], node_cmp); /* Add */ + ndpi_tsearch(newflow, &workflow->ndpi_flows_root[idx], ndpi_workflow_node_cmp); /* Add */ workflow->stats.ndpi_flow_count++; *src = newflow->src_id, *dst = newflow->dst_id; @@ -470,6 +430,7 @@ static unsigned int packet_processing(struct ndpi_workflow * workflow, return(0); } + /* Protocol already detected */ if(flow->detection_completed) return(0); flow->detected_protocol = ndpi_detection_process_packet(workflow->ndpi_struct, ndpi_flow, @@ -479,6 +440,7 @@ static unsigned int packet_processing(struct ndpi_workflow * workflow, if((flow->detected_protocol.protocol != NDPI_PROTOCOL_UNKNOWN) || ((proto == IPPROTO_UDP) && (flow->packets > 8)) || ((proto == IPPROTO_TCP) && (flow->packets > 10))) { + /* New protocol detected or give up */ flow->detection_completed = 1; if((flow->detected_protocol.protocol == NDPI_PROTOCOL_UNKNOWN) && (ndpi_flow->num_stun_udp_pkts > 0)) @@ -502,46 +464,19 @@ static unsigned int packet_processing(struct ndpi_workflow * workflow, snprintf(flow->ssl.server_certificate, sizeof(flow->ssl.server_certificate), "%s", flow->ndpi_flow->protos.ssl.server_certificate); } - if(flow->detected_protocol.protocol == NDPI_PROTOCOL_UNKNOWN) + if(flow->detected_protocol.protocol == NDPI_PROTOCOL_UNKNOWN) { flow->detected_protocol = ndpi_detection_giveup(workflow->ndpi_struct, flow->ndpi_flow); - + + if (workflow->__flow_giveup_callback != NULL) + workflow->__flow_giveup_callback(workflow, flow, workflow->__flow_giveup_udata); + } else { + if (workflow->__flow_detected_callback != NULL) + workflow->__flow_detected_callback(workflow, flow, workflow->__flow_detected_udata); + } + free_ndpi_flow_info(flow); - - /* TODO_EMA */ - /*if(verbose > 1) { - if(enable_protocol_guess) { - if(flow->detected_protocol.protocol == NDPI_PROTOCOL_UNKNOWN) { - flow->detected_protocol.protocol = node_guess_undetected_protocol(thread_id, flow), - flow->detected_protocol.master_protocol = NDPI_PROTOCOL_UNKNOWN; - } - } - - printFlow(thread_id, flow); - }*/ } - /* TODO_EMA use a callback */ - //~ if(live_capture) { - //~ if(ndpi_thread_info[thread_id].last_idle_scan_time + IDLE_SCAN_PERIOD < ndpi_thread_info[thread_id].last_time) { - //~ /* scan for idle flows */ - //~ ndpi_twalk(workflow->ndpi_flows_root[ndpi_thread_info[thread_id].idle_scan_idx], node_idle_scan_walker, &thread_id); - - //~ /* remove idle flows (unfortunately we cannot do this inline) */ - //~ while (ndpi_thread_info[thread_id].num_idle_flows > 0) { - - //~ /* search and delete the idle flow from the "ndpi_flow_root" (see struct reader thread) - here flows are the node of a b-tree */ - //~ ndpi_tdelete(ndpi_thread_info[thread_id].idle_flows[--ndpi_thread_info[thread_id].num_idle_flows], &workflow->ndpi_flows_root[ndpi_thread_info[thread_id].idle_scan_idx], node_cmp); - - //~ /* free the memory associated to idle flow in "idle_flows" - (see struct reader thread)*/ - //~ free_ndpi_flow_info(ndpi_thread_info[thread_id].idle_flows[ndpi_thread_info[thread_id].num_idle_flows]); - //~ ndpi_free(ndpi_thread_info[thread_id].idle_flows[ndpi_thread_info[thread_id].num_idle_flows]); - //~ } - - //~ if(++ndpi_thread_info[thread_id].idle_scan_idx == workflow->prefs.num_roots) ndpi_thread_info[thread_id].idle_scan_idx = 0; - //~ ndpi_thread_info[thread_id].last_idle_scan_time = ndpi_thread_info[thread_id].last_time; - //~ } - //~ } - return 0; } @@ -763,7 +698,8 @@ void ndpi_workflow_process_packet (struct ndpi_workflow * workflow, static u_int8_t cap_warning_used = 0; if(cap_warning_used == 0) { - if((!json_flag) && (!workflow->prefs.quiet_mode)) printf("\n\nWARNING: packet capture size is smaller than packet size, DETECTION MIGHT NOT WORK CORRECTLY\n\n"); + if(!workflow->prefs.quiet_mode) + NDPI_LOG(0, workflow.ndpi_struct, NDPI_LOG_DEBUG, "\n\nWARNING: packet capture size is smaller than packet size, DETECTION MIGHT NOT WORK CORRECTLY\n\n"); cap_warning_used = 1; } } @@ -783,7 +719,8 @@ void ndpi_workflow_process_packet (struct ndpi_workflow * workflow, workflow->stats.fragmented_count++; if(ipv4_frags_warning_used == 0) { - if((!json_flag) && (!workflow->prefs.quiet_mode)) printf("\n\nWARNING: IPv4 fragments are not handled by this demo (nDPI supports them)\n"); + if(!workflow->prefs.quiet_mode) + NDPI_LOG(0, workflow.ndpi_struct, NDPI_LOG_DEBUG, "\n\nWARNING: IPv4 fragments are not handled by this demo (nDPI supports them)\n"); ipv4_frags_warning_used = 1; } @@ -808,8 +745,8 @@ void ndpi_workflow_process_packet (struct ndpi_workflow * workflow, v4_warning: if(ipv4_warning_used == 0) { - if((!json_flag) && (!workflow->prefs.quiet_mode)) - printf("\n\nWARNING: only IPv4/IPv6 packets are supported in this demo (nDPI supports both IPv4 and IPv6), all other packets will be discarded\n\n"); + if(!workflow->prefs.quiet_mode) + NDPI_LOG(0, workflow.ndpi_struct, NDPI_LOG_DEBUG, "\n\nWARNING: only IPv4/IPv6 packets are supported in this demo (nDPI supports both IPv4 and IPv6), all other packets will be discarded\n\n"); ipv4_warning_used = 1; } workflow->stats.total_discarded_bytes += header->len; -- cgit v1.2.3 From 5386aede36b8abdd55a60a302913436859448377 Mon Sep 17 00:00:00 2001 From: emanuele-f Date: Wed, 20 Apr 2016 15:36:44 +0200 Subject: Integrate root flows free and little changes --- example/ndpiReader.c | 66 ++++++++++++----------------------------- src/include/ndpi_util.h | 22 +++++++++----- src/lib/ndpi_util.c | 78 +++++++++++++++++++++++++++++-------------------- 3 files changed, 79 insertions(+), 87 deletions(-) (limited to 'example/ndpiReader.c') diff --git a/example/ndpiReader.c b/example/ndpiReader.c index 0a1747088..43ce6d0c5 100644 --- a/example/ndpiReader.c +++ b/example/ndpiReader.c @@ -125,11 +125,6 @@ typedef struct ndpi_id { struct ndpi_id_struct *ndpi_id; // nDpi worker structure } ndpi_id_t; -static u_int32_t size_id_struct = 0; // ID tracking structure size - - -static u_int32_t size_flow_struct = 0; - static void help(u_int long_help) { printf("ndpiReader -i [-f ][-s ]\n" " [-p ][-l [-q][-d][-h][-t][-v ]\n" @@ -383,11 +378,11 @@ static void printFlow(u_int16_t thread_id, struct ndpi_flow_info *flow) { fprintf(out, "\t%s %s%s%s:%u <-> %s%s%s:%u ", ipProto2Name(flow->protocol), (flow->ip_version == 6) ? "[" : "", - flow->lower_name, + flow->lower_name, (flow->ip_version == 6) ? "]" : "", ntohs(flow->lower_port), (flow->ip_version == 6) ? "[" : "", - flow->upper_name, + flow->upper_name, (flow->ip_version == 6) ? "]" : "", ntohs(flow->upper_port)); @@ -472,24 +467,6 @@ static void printFlow(u_int16_t thread_id, struct ndpi_flow_info *flow) { /* ***************************************************** */ -static void free_ndpi_flow_info(struct ndpi_flow_info *flow) { - if(flow->ndpi_flow) { ndpi_free_flow(flow->ndpi_flow); flow->ndpi_flow = NULL; } - if(flow->src_id) { ndpi_free(flow->src_id); flow->src_id = NULL; } - if(flow->dst_id) { ndpi_free(flow->dst_id); flow->dst_id = NULL; } - -} - -/* ***************************************************** */ - -static void ndpi_flow_info_freer(void *node) { - struct ndpi_flow_info *flow = (struct ndpi_flow_info*)node; - - free_ndpi_flow_info(flow); - ndpi_free(flow); -} - -/* ***************************************************** */ - static void node_print_unknown_proto_walker(const void *node, ndpi_VISIT which, int depth, void *user_data) { struct ndpi_flow_info *flow = *(struct ndpi_flow_info**)node; u_int16_t thread_id = *((u_int16_t*)user_data); @@ -569,7 +546,7 @@ static void node_idle_scan_walker(const void *node, ndpi_VISIT which, int depth, if((flow->detected_protocol.protocol == NDPI_PROTOCOL_UNKNOWN) && !undetected_flows_deleted) undetected_flows_deleted = 1; - free_ndpi_flow_info(flow); + ndpi_free_flow_info_half(flow); ndpi_thread_info[thread_id].workflow->stats.ndpi_flow_count--; /* adding to a queue (we can't delete it from the tree inline ) */ @@ -584,7 +561,7 @@ static void on_protocol_discovered(struct ndpi_workflow * workflow, struct ndpi_flow_info * flow, void * udata) { const u_int16_t thread_id = *((u_int16_t *)udata); - + if(verbose > 1) { if(enable_protocol_guess) { if(flow->detected_protocol.protocol == NDPI_PROTOCOL_UNKNOWN) { @@ -638,7 +615,7 @@ static void debug_printf(u_int32_t protocol, void *id_struct, static void setupDetection(u_int16_t thread_id, pcap_t * pcap_handle) { NDPI_PROTOCOL_BITMASK all; - + struct ndpi_workflow_prefs prefs; memset(&prefs, 0, sizeof(prefs)); prefs.decode_tunnels = decode_tunnels; @@ -650,7 +627,7 @@ static void setupDetection(u_int16_t thread_id, pcap_t * pcap_handle) { memset(&ndpi_thread_info[thread_id], 0, sizeof(ndpi_thread_info[thread_id])); ndpi_thread_info[thread_id].workflow = ndpi_workflow_init(&prefs, pcap_handle, malloc_wrapper, free_wrapper, debug_printf); /* ndpi_thread_info[thread_id].workflow->ndpi_struct->http_dont_dissect_response = 1; */ - + ndpi_workflow_set_flow_detected_callback(ndpi_thread_info[thread_id].workflow, on_protocol_discovered, (void *)&thread_id); // enable all protocols @@ -669,13 +646,6 @@ static void setupDetection(u_int16_t thread_id, pcap_t * pcap_handle) { /* ***************************************************** */ static void terminateDetection(u_int16_t thread_id) { - int i; - - for(i=0; indpi_flows_root[i], ndpi_flow_info_freer); - ndpi_thread_info[thread_id].workflow->ndpi_flows_root[i] = NULL; - } - ndpi_workflow_free(ndpi_thread_info[thread_id].workflow); } @@ -810,7 +780,7 @@ static void printResults(u_int64_t tot_usec) { if(!quiet_mode) { printf("\nnDPI Memory statistics:\n"); printf("\tnDPI Memory (once): %-13s\n", formatBytes(sizeof(struct ndpi_detection_module_struct), buf, sizeof(buf))); - printf("\tFlow Memory (per flow): %-13s\n", formatBytes(size_flow_struct, buf, sizeof(buf))); + printf("\tFlow Memory (per flow): %-13s\n", formatBytes(sizeof(struct ndpi_flow_struct), buf, sizeof(buf))); printf("\tActual Memory: %-13s\n", formatBytes(current_ndpi_memory, buf, sizeof(buf))); printf("\tPeak Memory: %-13s\n", formatBytes(max_ndpi_memory, buf, sizeof(buf))); @@ -1112,7 +1082,7 @@ static pcap_t * openPcapFileOrDevice(u_int16_t thread_id, const u_char * pcap_fi signal(SIGALRM, sigproc); #endif } - + return pcap_handle; } @@ -1122,24 +1092,24 @@ static void pcap_packet_callback_checked(u_char *args, const struct pcap_pkthdr *header, const u_char *packet) { u_int16_t thread_id = *((u_int16_t*)args); - + /* allocate an exact size buffer to check overflows */ uint8_t *packet_checked = malloc(header->caplen); memcpy(packet_checked, packet, header->caplen); ndpi_workflow_process_packet(ndpi_thread_info[thread_id].workflow, header, packet_checked); - + if((capture_until != 0) && (header->ts.tv_sec >= capture_until)) { if(ndpi_thread_info[thread_id].workflow->pcap_handle != NULL) pcap_breakloop(ndpi_thread_info[thread_id].workflow->pcap_handle); return; } - + /* Check if capture is live or not */ if (!live_capture) { if (!pcap_start.tv_sec) pcap_start.tv_sec = header->ts.tv_sec, pcap_start.tv_usec = header->ts.tv_usec; pcap_end.tv_sec = header->ts.tv_sec, pcap_end.tv_usec = header->ts.tv_usec; } - + /* Idle flows cleanup */ if(live_capture) { if(ndpi_thread_info[thread_id].last_idle_scan_time + IDLE_SCAN_PERIOD < ndpi_thread_info[thread_id].workflow->last_time) { @@ -1148,22 +1118,22 @@ static void pcap_packet_callback_checked(u_char *args, /* remove idle flows (unfortunately we cannot do this inline) */ while (ndpi_thread_info[thread_id].num_idle_flows > 0) { - + /* search and delete the idle flow from the "ndpi_flow_root" (see struct reader thread) - here flows are the node of a b-tree */ ndpi_tdelete(ndpi_thread_info[thread_id].idle_flows[--ndpi_thread_info[thread_id].num_idle_flows], &ndpi_thread_info[thread_id].workflow->ndpi_flows_root[ndpi_thread_info[thread_id].idle_scan_idx], ndpi_workflow_node_cmp); - + /* free the memory associated to idle flow in "idle_flows" - (see struct reader thread)*/ - free_ndpi_flow_info(ndpi_thread_info[thread_id].idle_flows[ndpi_thread_info[thread_id].num_idle_flows]); + ndpi_free_flow_info_half(ndpi_thread_info[thread_id].idle_flows[ndpi_thread_info[thread_id].num_idle_flows]); ndpi_free(ndpi_thread_info[thread_id].idle_flows[ndpi_thread_info[thread_id].num_idle_flows]); } - + if(++ndpi_thread_info[thread_id].idle_scan_idx == ndpi_thread_info[thread_id].workflow->prefs.num_roots) ndpi_thread_info[thread_id].idle_scan_idx = 0; ndpi_thread_info[thread_id].last_idle_scan_time = ndpi_thread_info[thread_id].workflow->last_time; } } - + /* check for buffer changes */ if(memcmp(packet, packet_checked, header->caplen) != 0) printf("INTERNAL ERROR: ingress packet was nodified by nDPI: this should not happen [thread_id=%u, packetId=%lu]\n", @@ -1211,7 +1181,7 @@ void *processing_thread(void *_thread_id) { goto pcap_loop; } } - + return NULL; } diff --git a/src/include/ndpi_util.h b/src/include/ndpi_util.h index a23c5a0da..44f7be03d 100644 --- a/src/include/ndpi_util.h +++ b/src/include/ndpi_util.h @@ -24,13 +24,13 @@ /** * This module contains routines to help setup a simple nDPI program. - * + * * If you concern about performance or have to integrate nDPI in your * application, you could need to reimplement them yourself. - * + * * WARNING: this API is unstable! Use it at your own risk! */ - + #ifndef __NDPI_UTIL_H__ #define __NDPI_UTIL_H__ @@ -93,15 +93,15 @@ typedef void (*ndpi_workflow_callback_ptr) (struct ndpi_workflow *, struct ndpi_ typedef struct ndpi_workflow { u_int64_t last_time; - + struct ndpi_workflow_prefs prefs; struct ndpi_stats stats; - + ndpi_workflow_callback_ptr __flow_detected_callback; void * __flow_detected_udata; ndpi_workflow_callback_ptr __flow_giveup_callback; void * __flow_giveup_udata; - + /* outside referencies */ pcap_t *pcap_handle; @@ -116,14 +116,20 @@ struct ndpi_workflow * ndpi_workflow_init(const struct ndpi_workflow_prefs * pre void * (*malloc_wrapper)(size_t), void (*free_wrapper)(void*), ndpi_debug_function_ptr ndpi_debug_printf); - + void ndpi_workflow_free(struct ndpi_workflow * workflow); +/** Free flow_info ndpi support structures but not the flow_info itself + * + * TODO remove! Half freeing things is bad! + */ +void ndpi_free_flow_info_half(struct ndpi_flow_info *flow); + /** Process a @packet and update the @workflow. */ void ndpi_workflow_process_packet (struct ndpi_workflow * workflow, const struct pcap_pkthdr *header, const u_char *packet); - + /* flow callbacks: ndpi_flow_info will be freed right after */ static inline void ndpi_workflow_set_flow_detected_callback(struct ndpi_workflow * workflow, ndpi_workflow_callback_ptr callback, diff --git a/src/lib/ndpi_util.c b/src/lib/ndpi_util.c index 51d6757e1..36897cf2d 100644 --- a/src/lib/ndpi_util.c +++ b/src/lib/ndpi_util.c @@ -21,7 +21,7 @@ * along with nDPI. If not, see . * */ - + #include #ifdef WIN32 @@ -71,7 +71,7 @@ /* ***************************************************** */ -static void free_ndpi_flow_info(struct ndpi_flow_info *flow) { +void ndpi_free_flow_info_half(struct ndpi_flow_info *flow) { if(flow->ndpi_flow) { ndpi_free_flow(flow->ndpi_flow); flow->ndpi_flow = NULL; } if(flow->src_id) { ndpi_free(flow->src_id); flow->src_id = NULL; } if(flow->dst_id) { ndpi_free(flow->dst_id); flow->dst_id = NULL; } @@ -91,29 +91,45 @@ struct ndpi_workflow * ndpi_workflow_init(const struct ndpi_workflow_prefs * pre void * (*malloc_wrapper)(size_t), void (*free_wrapper)(void*), ndpi_debug_function_ptr ndpi_debug_printf) { - + /* TODO: just needed here to init ndpi malloc wrapper */ struct ndpi_detection_module_struct * module = ndpi_init_detection_module( prefs->detection_tick_resolution, malloc_wrapper, free_wrapper, ndpi_debug_printf); - + struct ndpi_workflow * workflow = ndpi_calloc(1, sizeof(struct ndpi_workflow)); - + removeme_free_wrapper = free_wrapper; - + workflow->pcap_handle = pcap_handle; workflow->prefs = *prefs; workflow->ndpi_struct = module; - + if(workflow->ndpi_struct == NULL) { NDPI_LOG(0, NULL, NDPI_LOG_ERROR, "global structure initialization failed\n"); exit(-1); } - + workflow->ndpi_flows_root = ndpi_calloc(workflow->prefs.num_roots, sizeof(void *)); return workflow; } +/* ***************************************************** */ + +static void ndpi_flow_info_freer(void *node) { + struct ndpi_flow_info *flow = (struct ndpi_flow_info*)node; + + ndpi_free_flow_info_half(flow); + ndpi_free(flow); +} + +/* ***************************************************** */ + void ndpi_workflow_free(struct ndpi_workflow * workflow) { + int i; + + for(i=0; iprefs.num_roots; i++) + ndpi_tdestroy(workflow->ndpi_flows_root[i], ndpi_flow_info_freer); + ndpi_exit_detection_module(workflow->ndpi_struct, removeme_free_wrapper); free(workflow->ndpi_flows_root); free(workflow); @@ -217,7 +233,7 @@ static struct ndpi_flow_info *get_ndpi_flow_info(struct ndpi_workflow * workflow // tcp *tcph = (struct ndpi_tcphdr *)l4; *sport = ntohs((*tcph)->source), *dport = ntohs((*tcph)->dest); - + if(iph->saddr < iph->daddr) { lower_port = (*tcph)->source, upper_port = (*tcph)->dest; *src_to_dst_direction = 1; @@ -235,7 +251,7 @@ static struct ndpi_flow_info *get_ndpi_flow_info(struct ndpi_workflow * workflow } } } - + tcp_len = ndpi_min(4*(*tcph)->doff, l4_packet_len); *payload = &l4[tcp_len]; *payload_len = ndpi_max(0, l4_packet_len-4*(*tcph)->doff); @@ -244,7 +260,7 @@ static struct ndpi_flow_info *get_ndpi_flow_info(struct ndpi_workflow * workflow workflow->stats.udp_count++; *udph = (struct ndpi_udphdr *)l4; - *sport = ntohs((*udph)->source), *dport = ntohs((*udph)->dest); + *sport = ntohs((*udph)->source), *dport = ntohs((*udph)->dest); *payload = &l4[sizeof(struct ndpi_udphdr)]; *payload_len = ndpi_max(0, l4_packet_len-sizeof(struct ndpi_udphdr)); @@ -259,7 +275,7 @@ static struct ndpi_flow_info *get_ndpi_flow_info(struct ndpi_workflow * workflow if(iph->saddr == iph->daddr) { if(lower_port > upper_port) { u_int16_t p = lower_port; - + lower_port = upper_port; upper_port = p; } @@ -406,17 +422,17 @@ static unsigned int packet_processing(struct ndpi_workflow * workflow, u_int16_t sport, dport, payload_len; u_int8_t *payload; u_int8_t src_to_dst_direction= 1; - + if(iph) flow = get_ndpi_flow_info(workflow, 4, vlan_id, iph, NULL, ip_offset, ipsize, ntohs(iph->tot_len) - (iph->ihl * 4), - &tcph, &udph, &sport, &dport, + &tcph, &udph, &sport, &dport, &src, &dst, &proto, &payload, &payload_len, &src_to_dst_direction); else flow = get_ndpi_flow_info6(workflow, vlan_id, iph6, ip_offset, - &tcph, &udph, &sport, &dport, + &tcph, &udph, &sport, &dport, &src, &dst, &proto, &payload, &payload_len, &src_to_dst_direction); @@ -436,7 +452,7 @@ static unsigned int packet_processing(struct ndpi_workflow * workflow, flow->detected_protocol = ndpi_detection_process_packet(workflow->ndpi_struct, ndpi_flow, iph ? (uint8_t *)iph : (uint8_t *)iph6, ipsize, time, src, dst); - + if((flow->detected_protocol.protocol != NDPI_PROTOCOL_UNKNOWN) || ((proto == IPPROTO_UDP) && (flow->packets > 8)) || ((proto == IPPROTO_TCP) && (flow->packets > 10))) { @@ -450,15 +466,15 @@ static unsigned int packet_processing(struct ndpi_workflow * workflow, if(flow->detected_protocol.protocol == NDPI_PROTOCOL_BITTORRENT) { int i, j, n = 0; - + for(i=0, j = 0; i<20; i++) { sprintf(&flow->bittorent_hash[j], "%02x", flow->ndpi_flow->bittorent_hash[i]); j += 2, n += flow->ndpi_flow->bittorent_hash[i]; } - + if(n == 0) flow->bittorent_hash[0] = '\0'; } - + if((proto == IPPROTO_TCP) && (flow->detected_protocol.protocol != NDPI_PROTOCOL_DNS)) { snprintf(flow->ssl.client_certificate, sizeof(flow->ssl.client_certificate), "%s", flow->ndpi_flow->protos.ssl.client_certificate); snprintf(flow->ssl.server_certificate, sizeof(flow->ssl.server_certificate), "%s", flow->ndpi_flow->protos.ssl.server_certificate); @@ -466,17 +482,17 @@ static unsigned int packet_processing(struct ndpi_workflow * workflow, if(flow->detected_protocol.protocol == NDPI_PROTOCOL_UNKNOWN) { flow->detected_protocol = ndpi_detection_giveup(workflow->ndpi_struct, flow->ndpi_flow); - + if (workflow->__flow_giveup_callback != NULL) workflow->__flow_giveup_callback(workflow, flow, workflow->__flow_giveup_udata); } else { if (workflow->__flow_detected_callback != NULL) workflow->__flow_detected_callback(workflow, flow, workflow->__flow_detected_udata); } - - free_ndpi_flow_info(flow); + + ndpi_free_flow_info_half(flow); } - + return 0; } @@ -487,7 +503,7 @@ void ndpi_workflow_process_packet (struct ndpi_workflow * workflow, /* * Declare pointers to packet headers */ - + /* --- Ethernet header --- */ const struct ndpi_ethhdr *ethernet; /* --- Ethernet II header --- */ @@ -576,7 +592,7 @@ void ndpi_workflow_process_packet (struct ndpi_workflow * workflow, ip_offset = sizeof(struct ndpi_chdlc); /* CHDLC_OFF = 4 */ type = ntohs(chdlc->proto_code); break; - + /* IEEE 802.3 Ethernet - 1 */ case DLT_EN10MB : ethernet = (struct ndpi_ethhdr *) &packet[eth_offset]; @@ -654,13 +670,13 @@ void ndpi_workflow_process_packet (struct ndpi_workflow * workflow, type = (packet[ip_offset+2] << 8) + packet[ip_offset+3]; ip_offset += 4; vlan_packet = 1; - } else if(type == MPLS_UNI || type == MPLS_MULTI) { + } else if(type == MPLS_UNI || type == MPLS_MULTI) { mpls = (struct ndpi_mpls_header *) &packet[ip_offset]; label = ntohl(mpls->label); /* label = ntohl(*((u_int32_t*)&packet[ip_offset])); */ workflow->stats.mpls_count++; type = ETH_P_IP, ip_offset += 4; - + while((label & 0x100) != 0x100) { ip_offset += 4; label = ntohl(mpls->label); @@ -676,13 +692,13 @@ void ndpi_workflow_process_packet (struct ndpi_workflow * workflow, else if(type == CISCO_D_PROTO) { cdp = (struct ndpi_cdp *) &packet[ip_offset]; cdp_pkts++; - } + } else if(type == PPPoE) { workflow->stats.pppoe_count++; type = ETH_P_IP; ip_offset += 8; } - + workflow->stats.vlan_count += vlan_packet; iph_check: @@ -787,13 +803,13 @@ void ndpi_workflow_process_packet (struct ndpi_workflow * workflow, if((version == 1) && (type == 0) && (encapsulates == 1)) { u_int8_t stop = 0; - + offset += 4; while((!stop) && (offset < header->caplen)) { u_int8_t tag_type = packet[offset]; u_int8_t tag_len; - + switch(tag_type) { case 0: /* PADDING Tag */ tag_len = 1; -- cgit v1.2.3 From 792d2fc79ac60aaf7af787a861fbc17312eec1c0 Mon Sep 17 00:00:00 2001 From: emanuele-f Date: Wed, 20 Apr 2016 16:20:50 +0200 Subject: Remove floating stack pointer causing segmentation fault --- example/ndpiReader.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'example/ndpiReader.c') diff --git a/example/ndpiReader.c b/example/ndpiReader.c index 43ce6d0c5..09e9a433a 100644 --- a/example/ndpiReader.c +++ b/example/ndpiReader.c @@ -560,7 +560,7 @@ static void node_idle_scan_walker(const void *node, ndpi_VISIT which, int depth, static void on_protocol_discovered(struct ndpi_workflow * workflow, struct ndpi_flow_info * flow, void * udata) { - const u_int16_t thread_id = *((u_int16_t *)udata); + const u_int16_t thread_id = (uintptr_t)udata; if(verbose > 1) { if(enable_protocol_guess) { @@ -628,7 +628,7 @@ static void setupDetection(u_int16_t thread_id, pcap_t * pcap_handle) { ndpi_thread_info[thread_id].workflow = ndpi_workflow_init(&prefs, pcap_handle, malloc_wrapper, free_wrapper, debug_printf); /* ndpi_thread_info[thread_id].workflow->ndpi_struct->http_dont_dissect_response = 1; */ - ndpi_workflow_set_flow_detected_callback(ndpi_thread_info[thread_id].workflow, on_protocol_discovered, (void *)&thread_id); + ndpi_workflow_set_flow_detected_callback(ndpi_thread_info[thread_id].workflow, on_protocol_discovered, (void *)(uintptr_t)thread_id); // enable all protocols NDPI_BITMASK_SET_ALL(all); -- cgit v1.2.3