diff options
Diffstat (limited to 'example/reader_util.c')
-rw-r--r-- | example/reader_util.c | 390 |
1 files changed, 354 insertions, 36 deletions
diff --git a/example/reader_util.c b/example/reader_util.c index d0ec31930..b8e21cce8 100644 --- a/example/reader_util.c +++ b/example/reader_util.c @@ -34,6 +34,8 @@ #else #include <unistd.h> #include <netinet/in.h> +#include <math.h> +#include <float.h> #endif #ifndef ETH_P_IP @@ -73,9 +75,158 @@ #include "ndpi_main.h" #include "reader_util.h" +#include "ndpi_classify.h" -extern u_int8_t enable_protocol_guess; +extern u_int8_t enable_protocol_guess, enable_joy_stats, enable_payload_analyzer; extern u_int8_t verbose, human_readeable_string_len; +extern u_int8_t max_num_udp_dissected_pkts /* 8 */, max_num_tcp_dissected_pkts /* 10 */; + +static u_int32_t flow_id = 0; + +/* ****************************************************** */ + +struct payload_stats { + u_int8_t *pattern; + u_int8_t pattern_len; + u_int16_t num_occurrencies; + UT_hash_handle hh; /* makes this structure hashable */ +}; + + +struct payload_stats *pstats = NULL; +u_int32_t max_num_packets_per_flow = 32; +u_int32_t max_packet_payload_dissection = 32; /* Full payload */ +u_int16_t min_pattern_len = 4; +u_int16_t max_pattern_len = 8; + + +void ndpi_analyze_payload(struct ndpi_flow_info *flow, + u_int8_t src_to_dst_direction, + u_int8_t *payload, + u_int16_t payload_len) { + struct payload_stats *ret; + u_int i; + +#ifdef DEBUG_PAYLOAD + for(i=0; i<payload_len; i++) + printf("%c", isprint(payload[i]) ? payload[i] : '.'); + printf("\n"); +#endif + + HASH_FIND(hh, pstats, payload, payload_len, ret); + if(ret == NULL) { + if((ret = (struct payload_stats*)calloc(1, sizeof(struct payload_stats))) == NULL) + return; /* OOM */ + + if((ret->pattern = (u_int8_t*)malloc(payload_len)) == NULL) { + free(ret); + return; + } + + memcpy(ret->pattern, payload, payload_len); + ret->pattern_len = payload_len; + ret->num_occurrencies = 1; + + HASH_ADD(hh, pstats, pattern[0], payload_len, ret); + +#ifdef DEBUG_PAYLOAD + printf("Added element [total: %u]\n", HASH_COUNT(pstats)); +#endif + } else { + ret->num_occurrencies++; + // printf("==> %u\n", ret->num_occurrencies); + } +} + + +void ndpi_payload_analyzer(struct ndpi_flow_info *flow, + u_int8_t src_to_dst_direction, + u_int8_t *payload, u_int16_t payload_len) { + u_int16_t i, j; + u_int16_t scan_len = ndpi_min(max_packet_payload_dissection, payload_len); + + if((flow->src2dst_pkt_count+flow->dst2src_pkt_count) < max_num_packets_per_flow) { +#ifdef DEBUG_PAYLOAD + printf("[hashval: %u][proto: %u][vlan: %u][%s:%u <-> %s:%u][direction: %s][payload_len: %u]\n", + flow->hashval, flow->protocol, flow->vlan_id, + flow->src_name, flow->src_port, + flow->dst_name, flow->dst_port, + src_to_dst_direction ? "s2d" : "d2s", + payload_len); +#endif + } else + return; + + for(i=0; i<scan_len; i++) { + for(j=min_pattern_len; j <= max_pattern_len; j++) { + if((i+j) < payload_len) { + ndpi_analyze_payload(flow, src_to_dst_direction, &payload[i], j); + ndpi_analyze_payload(flow, src_to_dst_direction, &payload[i], j); + } + } + } +} + +/* ***************************************************** */ + +static int payload_stats_sort_asc(void *_a, void *_b) { + struct payload_stats *a = (struct payload_stats *)_a; + struct payload_stats *b = (struct payload_stats *)_b; + + //return(a->num_occurrencies - b->num_occurrencies); + return(b->num_occurrencies - a->num_occurrencies); +} + +/* ***************************************************** */ + +void print_payload_stat(struct payload_stats *p) { + u_int i; + + printf("\t["); + + for(i=0; i<p->pattern_len; i++) { + printf("%c", isprint(p->pattern[i]) ? p->pattern[i] : '.'); + } + + printf("]"); + for(; i<16; i++) printf(" "); + printf("["); + + for(i=0; i<p->pattern_len; i++) { + printf("%s%02X", (i > 0) ? " " : "", isprint(p->pattern[i]) ? p->pattern[i] : '.'); + } + + printf("]"); + + for(; i<16; i++) printf(" "); + for(i=p->pattern_len; i<max_pattern_len; i++) printf(" "); + + printf("[len: %u][num_occurrencies: %u]\n", + p->pattern_len, p->num_occurrencies); +} + +/* ***************************************************** */ + +void ndpi_report_payload_stats() { + struct payload_stats *p, *tmp; + u_int num = 0, max_num = 25; + + printf("\n\nPayload Analysis\n"); + + HASH_SORT(pstats, payload_stats_sort_asc); + + HASH_ITER(hh, pstats, p, tmp) { + if(num <= max_num) + print_payload_stat(p); + + free(p->pattern); + HASH_DEL(pstats, p); + free(p); + num++; + } +} + + /* ***************************************************** */ @@ -273,6 +424,94 @@ int ndpi_workflow_node_cmp(const void *a, const void *b) { return(0); /* notreached */ } +/** + * \brief Update the byte count for the flow record. + * \param f Flow data + * \param x Data to use for update + * \param len Length of the data (in bytes) + * \return none + */ +static void +ndpi_flow_update_byte_count(struct ndpi_flow_info *flow, const void *x, + unsigned int len, u_int8_t src_to_dst_direction) { + const unsigned char *data = x; + u_int32_t i; + u_int32_t current_count = 0; + + /* + * implementation note: The spec says that 4000 octets is enough of a + * sample size to accurately reflect the byte distribution. Also, to avoid + * wrapping of the byte count at the 16-bit boundry, we stop counting once + * the 4000th octet has been seen for a flow. + */ + + /* octet count was already incremented before processing this payload */ + if (src_to_dst_direction) { + current_count = flow->src2dst_l4_bytes - len; + } else { + current_count = flow->dst2src_l4_bytes - len; + } + + if (current_count < ETTA_MIN_OCTETS) { + for (i=0; i<len; i++) { + if (src_to_dst_direction) { + flow->src2dst_byte_count[data[i]]++; + } else { + flow->dst2src_byte_count[data[i]]++; + } + current_count++; + if (current_count >= ETTA_MIN_OCTETS) { + break; + } + } + } +} + +/** + * \brief Update the byte distribution mean for the flow record. + * \param f Flow record + * \param x Data to use for update + * \param len Length of the data (in bytes) + * \return none + */ +static void +ndpi_flow_update_byte_dist_mean_var(ndpi_flow_info_t *flow, const void *x, + unsigned int len, u_int8_t src_to_dst_direction) { + const unsigned char *data = x; + double delta; + unsigned int i; + + for (i=0; i<len; i++) { + if (src_to_dst_direction) { + flow->src2dst_num_bytes += 1; + delta = ((double)data[i] - flow->src2dst_bd_mean); + flow->src2dst_bd_mean += delta/((double)flow->src2dst_num_bytes); + flow->src2dst_bd_variance += delta*((double)data[i] - flow->src2dst_bd_mean); + } else { + flow->dst2src_num_bytes += 1; + delta = ((double)data[i] - flow->dst2src_bd_mean); + flow->dst2src_bd_mean += delta/((double)flow->dst2src_num_bytes); + flow->dst2src_bd_variance += delta*((double)data[i] - flow->dst2src_bd_mean); + } + } +} + +float +ndpi_flow_get_byte_count_entropy(const uint32_t byte_count[256], + unsigned int num_bytes) +{ + int i; + float tmp, sum = 0.0; + + for (i=0; i<256; i++) { + tmp = (float) byte_count[i] / (float) num_bytes; + if (tmp > FLT_EPSILON) { + sum -= tmp * logf(tmp); + } + } + return sum / logf(2.0); +} + /* ***************************************************** */ static void patchIPv6Address(char *str) { @@ -309,11 +548,13 @@ static struct ndpi_flow_info *get_ndpi_flow_info(struct ndpi_workflow * workflow u_int8_t *proto, u_int8_t **payload, u_int16_t *payload_len, - u_int8_t *src_to_dst_direction) { + u_int8_t *src_to_dst_direction, + struct timeval when) { u_int32_t idx, l4_offset, hashval; struct ndpi_flow_info flow; void *ret; const u_int8_t *l3, *l4; + u_int32_t l4_data_len = 0XFEEDFACE; /* Note: to keep things simple (ndpiReader is just a demo app) @@ -363,6 +604,7 @@ static struct ndpi_flow_info *get_ndpi_flow_info(struct ndpi_workflow * workflow tcp_len = ndpi_min(4*(*tcph)->doff, l4_packet_len); *payload = (u_int8_t*)&l4[tcp_len]; *payload_len = ndpi_max(0, l4_packet_len-4*(*tcph)->doff); + l4_data_len = l4_packet_len - sizeof(struct ndpi_tcphdr); } else if(iph->protocol == IPPROTO_UDP && l4_packet_len >= 8) { // udp @@ -371,9 +613,11 @@ static struct ndpi_flow_info *get_ndpi_flow_info(struct ndpi_workflow * workflow *sport = ntohs((*udph)->source), *dport = ntohs((*udph)->dest); *payload = (u_int8_t*)&l4[sizeof(struct ndpi_udphdr)]; *payload_len = (l4_packet_len > sizeof(struct ndpi_udphdr)) ? l4_packet_len-sizeof(struct ndpi_udphdr) : 0; + l4_data_len = l4_packet_len - sizeof(struct ndpi_udphdr); } else { // non tcp/udp protocols *sport = *dport = 0; + l4_data_len = 0; } flow.protocol = iph->protocol, flow.vlan_id = vlan_id; @@ -418,6 +662,7 @@ static struct ndpi_flow_info *get_ndpi_flow_info(struct ndpi_workflow * workflow workflow->num_allocated_flows++; memset(newflow, 0, sizeof(struct ndpi_flow_info)); + newflow->flow_id = flow_id++; newflow->hashval = hashval; newflow->protocol = iph->protocol, newflow->vlan_id = vlan_id; newflow->src_ip = iph->saddr, newflow->dst_ip = iph->daddr; @@ -459,7 +704,15 @@ static struct ndpi_flow_info *get_ndpi_flow_info(struct ndpi_workflow * workflow workflow->stats.ndpi_flow_count++; *src = newflow->src_id, *dst = newflow->dst_id; - + newflow->src2dst_pkt_len[newflow->src2dst_pkt_count] = l4_packet_len; + newflow->src2dst_pkt_time[newflow->src2dst_pkt_count] = when; + if (newflow->src2dst_pkt_count == 0) { + newflow->src2dst_start = when; + } + newflow->src2dst_pkt_count++; + if (l4_data_len != 0XFEEDFACE) { + newflow->src2dst_opackets++; + } return newflow; } } else { @@ -485,6 +738,28 @@ static struct ndpi_flow_info *get_ndpi_flow_info(struct ndpi_workflow * workflow else *src = flow->dst_id, *dst = flow->src_id, *src_to_dst_direction = 0, flow->bidirectional = 1; } + if (src_to_dst_direction) { + if (flow->src2dst_pkt_count < MAX_NUM_PKTS) { + flow->src2dst_pkt_len[flow->src2dst_pkt_count] = l4_packet_len; + flow->src2dst_pkt_time[flow->src2dst_pkt_count] = when; + flow->src2dst_pkt_count++; + } + if (l4_data_len != 0XFEEDFACE) { + flow->src2dst_opackets++; + } + } else { + if (flow->dst2src_pkt_count < MAX_NUM_PKTS) { + flow->dst2src_pkt_len[flow->dst2src_pkt_count] = l4_packet_len; + flow->dst2src_pkt_time[flow->dst2src_pkt_count] = when; + if (flow->dst2src_pkt_count == 0) { + flow->dst2src_start = when; + } + flow->dst2src_pkt_count++; + } + if (l4_data_len != 0XFEEDFACE) { + flow->dst2src_opackets++; + } + } return flow; } } @@ -503,7 +778,8 @@ static struct ndpi_flow_info *get_ndpi_flow_info6(struct ndpi_workflow * workflo u_int8_t *proto, u_int8_t **payload, u_int16_t *payload_len, - u_int8_t *src_to_dst_direction) { + u_int8_t *src_to_dst_direction, + struct timeval when) { struct ndpi_iphdr iph; memset(&iph, 0, sizeof(iph)); @@ -523,12 +799,35 @@ static struct ndpi_flow_info *get_ndpi_flow_info6(struct ndpi_workflow * workflo ntohs(iph6->ip6_hdr.ip6_un1_plen), tcph, udph, sport, dport, src, dst, proto, payload, - payload_len, src_to_dst_direction)); + payload_len, src_to_dst_direction, when)); } /* ****************************************************** */ void process_ndpi_collected_info(struct ndpi_workflow * workflow, struct ndpi_flow_info *flow) { + + if(enable_joy_stats) { + /* Update SPLT scores. */ + + if(flow->bidirectional) + flow->score = ndpi_classify(flow->src2dst_pkt_len, flow->src2dst_pkt_time, + flow->dst2src_pkt_len, flow->dst2src_pkt_time, + flow->src2dst_start, flow->dst2src_start, + MAX_NUM_PKTS, flow->src_port, flow->dst_port, + flow->src2dst_packets, flow->dst2src_packets, + flow->src2dst_opackets, flow->dst2src_opackets, + flow->src2dst_l4_bytes, flow->dst2src_l4_bytes, 1, + flow->src2dst_byte_count, flow->dst2src_byte_count); + else + flow->score = ndpi_classify(flow->src2dst_pkt_len, flow->src2dst_pkt_time, + NULL, NULL, flow->src2dst_start, flow->src2dst_start, + MAX_NUM_PKTS, flow->src_port, flow->dst_port, + flow->src2dst_packets, 0, + flow->src2dst_opackets, 0, + flow->src2dst_l4_bytes, 0, 1, + flow->src2dst_byte_count, NULL); + } + if(!flow->ndpi_flow) return; snprintf(flow->host_server_name, sizeof(flow->host_server_name), "%s", @@ -559,27 +858,33 @@ void process_ndpi_collected_info(struct ndpi_workflow * workflow, struct ndpi_fl if(flow->detected_protocol.app_protocol != NDPI_PROTOCOL_DNS) { /* SSH */ if(flow->detected_protocol.app_protocol == NDPI_PROTOCOL_SSH) { - snprintf(flow->ssh_ssl.client_info, sizeof(flow->ssh_ssl.client_info), "%s", + snprintf(flow->ssh_tls.client_info, sizeof(flow->ssh_tls.client_info), "%s", flow->ndpi_flow->protos.ssh.client_signature); - snprintf(flow->ssh_ssl.server_info, sizeof(flow->ssh_ssl.server_info), "%s", + snprintf(flow->ssh_tls.server_info, sizeof(flow->ssh_tls.server_info), "%s", flow->ndpi_flow->protos.ssh.server_signature); + snprintf(flow->ssh_tls.client_hassh, sizeof(flow->ssh_tls.client_hassh), "%s", + flow->ndpi_flow->protos.ssh.hassh_client); + snprintf(flow->ssh_tls.server_hassh, sizeof(flow->ssh_tls.server_hassh), "%s", + flow->ndpi_flow->protos.ssh.hassh_server); } - /* SSL */ - else if((flow->detected_protocol.app_protocol == NDPI_PROTOCOL_SSL) - || (flow->detected_protocol.master_protocol == NDPI_PROTOCOL_SSL)) { - flow->ssh_ssl.ssl_version = flow->ndpi_flow->protos.stun_ssl.ssl.ssl_version; - snprintf(flow->ssh_ssl.client_info, sizeof(flow->ssh_ssl.client_info), "%s", + /* TLS */ + else if((flow->detected_protocol.app_protocol == NDPI_PROTOCOL_TLS) + || (flow->detected_protocol.master_protocol == NDPI_PROTOCOL_TLS) + || (flow->ndpi_flow->protos.stun_ssl.ssl.ja3_client[0] != '\0') + ) { + flow->ssh_tls.ssl_version = flow->ndpi_flow->protos.stun_ssl.ssl.ssl_version; + snprintf(flow->ssh_tls.client_info, sizeof(flow->ssh_tls.client_info), "%s", flow->ndpi_flow->protos.stun_ssl.ssl.client_certificate); - snprintf(flow->ssh_ssl.server_info, sizeof(flow->ssh_ssl.server_info), "%s", + snprintf(flow->ssh_tls.server_info, sizeof(flow->ssh_tls.server_info), "%s", flow->ndpi_flow->protos.stun_ssl.ssl.server_certificate); - snprintf(flow->ssh_ssl.server_organization, sizeof(flow->ssh_ssl.server_organization), "%s", + snprintf(flow->ssh_tls.server_organization, sizeof(flow->ssh_tls.server_organization), "%s", flow->ndpi_flow->protos.stun_ssl.ssl.server_organization); - snprintf(flow->ssh_ssl.ja3_client, sizeof(flow->ssh_ssl.ja3_client), "%s", + snprintf(flow->ssh_tls.ja3_client, sizeof(flow->ssh_tls.ja3_client), "%s", flow->ndpi_flow->protos.stun_ssl.ssl.ja3_client); - snprintf(flow->ssh_ssl.ja3_server, sizeof(flow->ssh_ssl.ja3_server), "%s", + snprintf(flow->ssh_tls.ja3_server, sizeof(flow->ssh_tls.ja3_server), "%s", flow->ndpi_flow->protos.stun_ssl.ssl.ja3_server); - flow->ssh_ssl.server_unsafe_cipher = flow->ndpi_flow->protos.stun_ssl.ssl.server_unsafe_cipher; - flow->ssh_ssl.server_cipher = flow->ndpi_flow->protos.stun_ssl.ssl.server_cipher; + flow->ssh_tls.server_unsafe_cipher = flow->ndpi_flow->protos.stun_ssl.ssl.server_unsafe_cipher; + flow->ssh_tls.server_cipher = flow->ndpi_flow->protos.stun_ssl.ssl.server_cipher; } } @@ -613,7 +918,8 @@ static struct ndpi_proto packet_processing(struct ndpi_workflow * workflow, u_int16_t ip_offset, u_int16_t ipsize, u_int16_t rawsize, const struct pcap_pkthdr *header, - const u_char *packet) { + const u_char *packet, + struct timeval when) { struct ndpi_id_struct *src, *dst; struct ndpi_flow_info *flow = NULL; struct ndpi_flow_struct *ndpi_flow = NULL; @@ -631,12 +937,12 @@ static struct ndpi_proto packet_processing(struct ndpi_workflow * workflow, ntohs(iph->tot_len) - (iph->ihl * 4), &tcph, &udph, &sport, &dport, &src, &dst, &proto, - &payload, &payload_len, &src_to_dst_direction); + &payload, &payload_len, &src_to_dst_direction, when); else flow = get_ndpi_flow_info6(workflow, vlan_id, iph6, ip_offset, &tcph, &udph, &sport, &dport, &src, &dst, &proto, - &payload, &payload_len, &src_to_dst_direction); + &payload, &payload_len, &src_to_dst_direction, when); if(flow != NULL) { workflow->stats.ip_packet_count++; @@ -644,22 +950,34 @@ static struct ndpi_proto packet_processing(struct ndpi_workflow * workflow, workflow->stats.total_ip_bytes += rawsize; ndpi_flow = flow->ndpi_flow; - if(src_to_dst_direction) + if(src_to_dst_direction) { flow->src2dst_packets++, flow->src2dst_bytes += rawsize; - else + flow->src2dst_l4_bytes += payload_len; + } else { flow->dst2src_packets++, flow->dst2src_bytes += rawsize; + flow->dst2src_l4_bytes += payload_len; + } + + if(enable_payload_analyzer && (payload_len > 0)) + ndpi_payload_analyzer(flow, src_to_dst_direction, payload, payload_len); + + if(enable_joy_stats) { + /* Update BD, distribution and mean. */ + ndpi_flow_update_byte_count(flow, payload, payload_len, src_to_dst_direction); + ndpi_flow_update_byte_dist_mean_var(flow, payload, payload_len, src_to_dst_direction); + } flow->last_seen = time; if(!flow->has_human_readeable_strings) { u_int8_t skip = 0; - + if((proto == IPPROTO_TCP) && ( - (flow->detected_protocol.app_protocol == NDPI_PROTOCOL_SSL) - || (flow->detected_protocol.master_protocol == NDPI_PROTOCOL_SSL) + (flow->detected_protocol.app_protocol == NDPI_PROTOCOL_TLS) + || (flow->detected_protocol.master_protocol == NDPI_PROTOCOL_TLS) || (flow->detected_protocol.app_protocol == NDPI_PROTOCOL_SSH) - || (flow->detected_protocol.master_protocol == NDPI_PROTOCOL_SSH)) + || (flow->detected_protocol.master_protocol == NDPI_PROTOCOL_SSH)) ) { if((flow->src2dst_packets+flow->dst2src_packets) < 10 /* MIN_NUM_ENCRYPT_SKIP_PACKETS */) skip = 1; @@ -667,7 +985,7 @@ static struct ndpi_proto packet_processing(struct ndpi_workflow * workflow, if(!skip) { char outbuf[64] = { '\0' }; - + if(ndpi_has_human_readeable_string(workflow->ndpi_struct, (char*)packet, header->caplen, human_readeable_string_len, flow->human_readeable_string_buffer, @@ -677,10 +995,10 @@ static struct ndpi_proto packet_processing(struct ndpi_workflow * workflow, } else { if((proto == IPPROTO_TCP) && ( - (flow->detected_protocol.app_protocol == NDPI_PROTOCOL_SSL) - || (flow->detected_protocol.master_protocol == NDPI_PROTOCOL_SSL) + (flow->detected_protocol.app_protocol == NDPI_PROTOCOL_TLS) + || (flow->detected_protocol.master_protocol == NDPI_PROTOCOL_TLS) || (flow->detected_protocol.app_protocol == NDPI_PROTOCOL_SSH) - || (flow->detected_protocol.master_protocol == NDPI_PROTOCOL_SSH)) + || (flow->detected_protocol.master_protocol == NDPI_PROTOCOL_SSH)) ) flow->has_human_readeable_strings = 0; } @@ -691,8 +1009,8 @@ static struct ndpi_proto packet_processing(struct ndpi_workflow * workflow, if(!flow->detection_completed) { u_int enough_packets = - (((proto == IPPROTO_UDP) && ((flow->src2dst_packets + flow->dst2src_packets) > 8)) - || ((proto == IPPROTO_TCP) && ((flow->src2dst_packets + flow->dst2src_packets) > 10))) ? 1 : 0; + (((proto == IPPROTO_UDP) && ((flow->src2dst_packets + flow->dst2src_packets) > max_num_udp_dissected_pkts)) + || ((proto == IPPROTO_TCP) && ((flow->src2dst_packets + flow->dst2src_packets) > max_num_tcp_dissected_pkts))) ? 1 : 0; flow->detected_protocol = ndpi_detection_process_packet(workflow->ndpi_struct, ndpi_flow, iph ? (uint8_t *)iph : (uint8_t *)iph6, @@ -700,7 +1018,7 @@ static struct ndpi_proto packet_processing(struct ndpi_workflow * workflow, if(enough_packets || (flow->detected_protocol.app_protocol != NDPI_PROTOCOL_UNKNOWN)) { if((!enough_packets) - && (flow->detected_protocol.master_protocol == NDPI_PROTOCOL_SSL) + && (flow->detected_protocol.master_protocol == NDPI_PROTOCOL_TLS) && (flow->ndpi_flow->protos.stun_ssl.ssl.ja3_server[0] == '\0')) ; /* Wait for JA3S certificate */ else { @@ -985,7 +1303,7 @@ iph_check: proto = options[0]; ip_len += 8 * (options[1] + 1); } - + iph = NULL; } else { static u_int8_t ipv4_warning_used = 0; @@ -1070,7 +1388,7 @@ iph_check: /* process the packet */ return(packet_processing(workflow, time, vlan_id, iph, iph6, ip_offset, header->caplen - ip_offset, - header->caplen, header, packet)); + header->caplen, header, packet, header->ts)); } /* ********************************************************** */ |