diff options
| -rw-r--r-- | example/ndpiReader.c | 39 | ||||
| -rw-r--r-- | example/reader_util.c | 69 |
2 files changed, 80 insertions, 28 deletions
diff --git a/example/ndpiReader.c b/example/ndpiReader.c index ce86c6fcd..b744a8708 100644 --- a/example/ndpiReader.c +++ b/example/ndpiReader.c @@ -91,6 +91,10 @@ static ndpi_serialization_format serialization_format = ndpi_serialization_forma static char* domain_to_check = NULL; static char* ip_port_to_check = NULL; static u_int8_t ignore_vlanid = 0; + +FILE *fingerprint_fp = NULL; /**< for flow fingerprint export */ + + /** User preferences **/ u_int8_t enable_realtime_output = 0, enable_protocol_guess = NDPI_GIVEUP_GUESS_BY_PORT | NDPI_GIVEUP_GUESS_BY_IP, enable_payload_analyzer = 0, num_bin_clusters = 0, extcap_exit = 0; u_int8_t verbose = 0, enable_flow_stats = 0; @@ -617,7 +621,7 @@ static void help(u_int long_help) { "-i <file|device> " #endif "[-f <filter>][-s <duration>][-m <duration>][-b <num bin clusters>]\n" - " [-p <protos>][-l <loops> [-q][-d][-h][-H][-D][-e <len>][-E][-t][-v <level>]\n" + " [-p <protos>][-l <loops> [-q][-d][-h][-H][-D][-e <len>][-E <path>][-t][-v <level>]\n" " [-n <threads>][-w <file>][-c <file>][-C <file>][-j <file>][-x <file>]\n" " [-r <file>][-R][-j <file>][-S <file>][-T <num>][-U <num>] [-x <domain>]\n" " [-a <mode>][-B proto_list]\n\n" @@ -657,6 +661,7 @@ static void help(u_int long_help) { " | Default: %u:%u:%u:%u:%u\n" " -c <path> | Load custom categories from the specified file\n" " -C <path> | Write output in CSV format on the specified file\n" + " -E <path> | Write flow fingerprints on the specified file\n" " -r <path> | Load risky domain file\n" " -R | Print detected realtime protocols\n" " -j <path> | Load malicious JA3 fingeprints\n" @@ -938,6 +943,14 @@ void extcap_capture(int datalink_type) { /* ********************************** */ +void printFingerprintHeader() { + if(!fingerprint_fp) return; + + fprintf(fingerprint_fp, "#protocol|src_ip|dst_ip|dst_port|family|fingerprint\n"); +} + +/* ********************************** */ + void printCSVHeader() { if(!csv_fp) return; @@ -1074,7 +1087,7 @@ static void parseOptions(int argc, char **argv) { #endif while((opt = getopt_long(argc, argv, - "a:Ab:B:e:c:C:dDFf:g:G:i:Ij:k:K:S:hHp:pP:l:r:Rs:tu:v:V:n:rp:x:X:w:q0123:456:7:89:m:MT:U:", + "a:Ab:B:e:E:c:C:dDFf:g:G:i:Ij:k:K:S:hHp:pP:l:r:Rs:tu:v:V:n:rp:x:X:w:q0123:456:7:89:m:MT:U:", longopts, &option_idx)) != EOF) { #ifdef DEBUG_TRACE if(trace) fprintf(trace, " #### Handling option -%c [%s] #### \n", opt, optarg ? optarg : ""); @@ -1110,6 +1123,19 @@ static void parseOptions(int argc, char **argv) { human_readeable_string_len = atoi(optarg); break; + case 'E': + errno = 0; + if((fingerprint_fp = fopen(optarg, "w")) == NULL) { + printf("Unable to write on fingerprint file %s: %s\n", optarg, strerror(errno)); + exit(1); + } + + if(reader_add_cfg("tls", "metadata.ja4r_fingerprint", "1", 1) == -1) { + printf("Unable to enable JA4r fingerprints\n"); + exit(1); + } + break; + case 'i': case '3': _pcap_file[0] = optarg; @@ -1433,9 +1459,9 @@ static void parseOptions(int argc, char **argv) { if(extcap_exit) exit(0); - if(csv_fp) - printCSVHeader(); - + printCSVHeader(); + printFingerprintHeader(); + #ifndef USE_DPDK if(do_extcap_capture) { quiet_mode = 1; @@ -6463,7 +6489,8 @@ int main(int argc, char **argv) { if(extcap_dumper) pcap_dump_close(extcap_dumper); if(extcap_fifo_h) pcap_close(extcap_fifo_h); if(enable_malloc_bins) ndpi_free_bin(&malloc_bins); - if(csv_fp) fclose(csv_fp); + if(csv_fp) fclose(csv_fp); + if(fingerprint_fp) fclose(fingerprint_fp); ndpi_free(_disabled_protocols); diff --git a/example/reader_util.c b/example/reader_util.c index 014574503..d2b60859d 100644 --- a/example/reader_util.c +++ b/example/reader_util.c @@ -78,6 +78,7 @@ extern u_int8_t enable_flow_stats, enable_payload_analyzer; extern u_int8_t verbose, human_readeable_string_len; extern u_int8_t max_num_udp_dissected_pkts /* 24 */, max_num_tcp_dissected_pkts /* 80 */; static u_int32_t flow_id = 0; +extern FILE *fingerprint_fp; u_int8_t enable_doh_dot_detection = 0; @@ -462,7 +463,7 @@ static void ndpi_free_flow_tls_data(struct ndpi_flow_info *flow) { ndpi_free(flow->dhcp_fingerprint); flow->dhcp_fingerprint = NULL; } - + if(flow->dhcp_class_ident) { ndpi_free(flow->dhcp_class_ident); flow->dhcp_class_ident = NULL; @@ -477,7 +478,7 @@ static void ndpi_free_flow_tls_data(struct ndpi_flow_info *flow) { ndpi_free(flow->telnet.username); flow->telnet.username = NULL; } - + if(flow->telnet.password) { ndpi_free(flow->telnet.password); flow->telnet.password = NULL; @@ -784,7 +785,7 @@ static struct ndpi_flow_info *get_ndpi_flow_info(struct ndpi_workflow * workflow flow.protocol = iph->protocol, flow.vlan_id = vlan_id; flow.src_ip = iph->saddr, flow.dst_ip = iph->daddr; flow.src_port = htons(*sport), flow.dst_port = htons(*dport); - flow.hashval = hashval = flow.protocol + ntohl(flow.src_ip) + ntohl(flow.dst_ip) + flow.hashval = hashval = flow.protocol + ntohl(flow.src_ip) + ntohl(flow.dst_ip) + ntohs(flow.src_port) + ntohs(flow.dst_port); #if 0 @@ -1041,10 +1042,31 @@ u_int8_t plen2slot(u_int16_t plen) { /* ****************************************************** */ +static void dump_raw_fingerprint(struct ndpi_workflow * workflow, + struct ndpi_flow_info *flow, + char *fingerprint_family, + char *fingerprint) { + char buf[64]; + + fprintf(fingerprint_fp, "%u|%s|%s|%u|%s|%s|%s\n", + flow->protocol,flow->src_name, flow->dst_name, ntohs(flow->dst_port), + ndpi_protocol2name(workflow->ndpi_struct, flow->detected_protocol, buf, sizeof(buf)), + fingerprint_family, fingerprint); +} + +/* ****************************************************** */ + +static void dump_flow_fingerprint(struct ndpi_workflow * workflow, struct ndpi_flow_info *flow) { + if(flow->ndpi_flow->protos.tls_quic.ja4_client_raw != NULL) + dump_raw_fingerprint(workflow, flow, "JA4r", flow->ndpi_flow->protos.tls_quic.ja4_client_raw); +} + +/* ****************************************************** */ + void process_ndpi_collected_info(struct ndpi_workflow * workflow, struct ndpi_flow_info *flow) { u_int i, is_quic = 0; char out[128], *s; - + if(!flow->ndpi_flow) return; flow->info_type = INFO_INVALID; @@ -1052,8 +1074,8 @@ void process_ndpi_collected_info(struct ndpi_workflow * workflow, struct ndpi_fl s = ndpi_get_flow_risk_info(flow->ndpi_flow, out, sizeof(out), 0 /* text */); if(s != NULL) - flow->risk_str = ndpi_strdup(s); - + flow->risk_str = ndpi_strdup(s); + flow->confidence = flow->ndpi_flow->confidence; flow->fpc = flow->ndpi_flow->fpc; @@ -1067,7 +1089,7 @@ void process_ndpi_collected_info(struct ndpi_workflow * workflow, struct ndpi_fl ndpi_snprintf(flow->mining.currency, sizeof(flow->mining.currency), "%s", flow->ndpi_flow->protos.mining.currency); } - + flow->risk = flow->ndpi_flow->risk; if(is_ndpi_proto(flow, NDPI_PROTOCOL_DHCP)) { @@ -1084,7 +1106,7 @@ void process_ndpi_collected_info(struct ndpi_workflow * workflow, struct ndpi_fl if(flow->ndpi_flow->protos.bittorrent.hash[0] != '\0') { u_int avail = sizeof(flow->ndpi_flow->protos.bittorrent.hash) * 2 + 1; flow->bittorent_hash = ndpi_malloc(avail); - + if(flow->bittorent_hash) { for(i=0, j = 0; i < sizeof(flow->ndpi_flow->protos.bittorrent.hash); i++) { @@ -1093,7 +1115,7 @@ void process_ndpi_collected_info(struct ndpi_workflow * workflow, struct ndpi_fl j += 2; } - + flow->bittorent_hash[j] = '\0'; } } @@ -1125,7 +1147,7 @@ void process_ndpi_collected_info(struct ndpi_workflow * workflow, struct ndpi_fl /* SERVICE_LOCATION */ else if(is_ndpi_proto(flow, NDPI_PROTOCOL_SERVICE_LOCATION)) { size_t i; - + flow->info_type = INFO_GENERIC; flow->info[0] = 0; if (flow->ndpi_flow->protos.slp.url_count > 0) @@ -1133,7 +1155,7 @@ void process_ndpi_collected_info(struct ndpi_workflow * workflow, struct ndpi_fl for (i = 0; i < flow->ndpi_flow->protos.slp.url_count; ++i) { size_t length = strlen(flow->info); - + strncat(flow->info + length, flow->ndpi_flow->protos.slp.url[i], sizeof(flow->info) - length); length = strlen(flow->info); @@ -1325,7 +1347,7 @@ void process_ndpi_collected_info(struct ndpi_workflow * workflow, struct ndpi_fl memcpy(&flow->stun.other_address, &flow->ndpi_flow->stun.other_address, sizeof(ndpi_address_port)); flow->multimedia_flow_type = flow->ndpi_flow->flow_multimedia_type; - + /* HTTP metadata are "global" not in `flow->ndpi_flow->protos` union; for example, we can have HTTP/BitTorrent and in that case we want to export also HTTP attributes */ if(is_ndpi_proto(flow, NDPI_PROTOCOL_HTTP) @@ -1358,18 +1380,21 @@ void process_ndpi_collected_info(struct ndpi_workflow * workflow, struct ndpi_fl LOG(NDPI_LOG_ERROR, "flow2json failed\n"); return; } - + ndpi_serialize_string_uint32(&flow->ndpi_flow_serializer, "detection_completed", flow->detection_completed); ndpi_serialize_string_uint32(&flow->ndpi_flow_serializer, "check_extra_packets", flow->check_extra_packets); } - if(flow->detection_completed && (!flow->check_extra_packets)) { + if(flow->detection_completed && (!flow->check_extra_packets)) { flow->flow_payload = flow->ndpi_flow->flow_payload, flow->flow_payload_len = flow->ndpi_flow->flow_payload_len; flow->ndpi_flow->flow_payload = NULL; /* We'll free the memory */ if(workflow->flow_callback != NULL) workflow->flow_callback(workflow, flow, workflow->flow_callback_userdata); + if(fingerprint_fp) + dump_flow_fingerprint(workflow, flow); + ndpi_free_flow_info_half(flow); } } @@ -1693,7 +1718,7 @@ static struct ndpi_proto packet_processing(struct ndpi_workflow * workflow, } malloc_size_stats = 0; } - + #if 0 if(flow->risk != 0) { FILE *r = fopen("/tmp/e", "a"); @@ -1746,7 +1771,7 @@ static bool ndpi_is_valid_vxlan(const struct pcap_pkthdr *header, const u_char * struct ndpi_udphdr *udp = (struct ndpi_udphdr *)&packet[ip_offset+ip_len]; u_int offset = ip_offset + ip_len + sizeof(struct ndpi_udphdr); /** - * rfc-7348 + * rfc-7348 * VXLAN Header: This is an 8-byte field that has: - Flags (8 bits): where the I flag MUST be set to 1 for a valid @@ -1784,8 +1809,8 @@ static inline u_int ndpi_skip_vxlan(u_int16_t ip_offset, u_int16_t ip_len){ return ip_offset + ip_len + sizeof(struct ndpi_udphdr) + sizeof(struct ndpi_vxlanhdr); } -static uint32_t ndpi_is_valid_gre_tunnel(const struct pcap_pkthdr *header, - const u_char *packet, const u_int16_t ip_offset, +static uint32_t ndpi_is_valid_gre_tunnel(const struct pcap_pkthdr *header, + const u_char *packet, const u_int16_t ip_offset, const u_int16_t ip_len) { if(header->caplen < ip_offset + ip_len + sizeof(struct ndpi_gre_basehdr)) return 0; /* Too short for GRE header*/ @@ -1829,7 +1854,7 @@ static uint32_t ndpi_is_valid_gre_tunnel(const struct pcap_pkthdr *header, return 0; if(NDPI_GRE_IS_STRICT(grehdr->flags)) return 0; - if(grehdr->protocol != NDPI_GRE_PROTO_PPP) + if(grehdr->protocol != NDPI_GRE_PROTO_PPP) return 0; /*key field*/ if(header->caplen < offset + 4) @@ -2364,11 +2389,11 @@ struct ndpi_proto ndpi_workflow_process_packet(struct ndpi_workflow * workflow, if((offset = ndpi_is_valid_gre_tunnel(header, packet, ip_offset, ip_len))) { tunnel_type = ndpi_gre_tunnel; struct ndpi_gre_basehdr *grehdr = (struct ndpi_gre_basehdr*)&packet[ip_offset + ip_len]; - if(grehdr->protocol == ntohs(ETH_P_IP) || grehdr->protocol == ntohs(ETH_P_IPV6)) { + if(grehdr->protocol == ntohs(ETH_P_IP) || grehdr->protocol == ntohs(ETH_P_IPV6)) { ip_offset = offset; - goto iph_check; + goto iph_check; } else if(grehdr->protocol == NDPI_GRE_PROTO_PPP) { // ppp protocol - ip_offset = offset + NDPI_PPP_HDRLEN; + ip_offset = offset + NDPI_PPP_HDRLEN; goto iph_check; } else { eth_offset = offset; |