Added nDPIsrvd-analysed to generate CSV files from analyse events.

 * nDPIsrvd.h: iterate over JSON arrays
 * nDPId: calculate l3 payload packet entropies for analysis

Signed-off-by: Toni Uhlig <matzeton@googlemail.com>

1 files changed, 6 insertions, 6 deletions

diff --git a/test/results/nfsv2.pcap.out b/test/results/nfsv2.pcap.out
index bb812f091..937d04bf9 100644
--- a/test/results/nfsv2.pcap.out
+++ b/test/results/nfsv2.pcap.out
@@ -21,7 +21,7 @@
 00859{"flow_event_id":7,"flow_event_name":"detected","thread_id":0,"packet_id":9,"source":"nfsv2.pcap","alias":"nDPId-test","flow_id":5,"flow_state":"info","flow_src_packets_processed":1,"flow_dst_packets_processed":0,"flow_first_seen":944207338490000,"flow_src_last_pkt_time":944207338490000,"flow_dst_last_pkt_time":944207338490000,"flow_idle_time":200000000,"flow_src_min_l4_payload_len":124,"flow_dst_min_l4_payload_len":0,"flow_src_max_l4_payload_len":124,"flow_dst_max_l4_payload_len":0,"flow_src_tot_l4_payload_len":124,"flow_dst_tot_l4_payload_len":0,"midstream":0,"thread_ts_usec":944207338490000,"l3_proto":"ip4","src_ip":"139.25.22.2","dst_ip":"139.25.22.102","src_port":1023,"dst_port":2049,"l4_proto":"udp","ndpi": {"confidence": {"6":"DPI"},"proto":"NFS","proto_id":"11","encrypted":0,"breed":"Acceptable","category_id":4,"category":"DataTransfer"}}
 00618{"packet_event_id":2,"packet_event_name":"packet-flow","thread_id":0,"packet_id":10,"source":"nfsv2.pcap","alias":"nDPId-test","flow_id":5,"flow_packet_id":2,"flow_src_last_pkt_time":944207338490000,"flow_dst_last_pkt_time":944207338490000,"flow_idle_time":200000000,"pkt_oversize":false,"pkt_caplen":138,"pkt_type":2048,"pkt_l3_offset":14,"pkt_l4_offset":34,"pkt_len":138,"pkt_l4_len":104,"thread_ts_usec":944207338490000,"pkt":"AMCV4Bm+AMCV+E3TCABFAAB8jl8AAP8R6naLGRZmixkWAggBA\/8AaNSdXh0LlAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAEHtAAAAAgAAAAAAAAABAAAAYAAAQAAAAAAAAAAAAAAQEIUAALJaOEd1QgAFMCA4R3VCAAd6EDhHdUIAB3oQ"}
 00659{"packet_event_id":2,"packet_event_name":"packet-flow","thread_id":0,"packet_id":11,"source":"nfsv2.pcap","alias":"nDPId-test","flow_id":5,"flow_packet_id":3,"flow_src_last_pkt_time":944207338490000,"flow_dst_last_pkt_time":944207338490000,"flow_idle_time":200000000,"pkt_oversize":false,"pkt_caplen":166,"pkt_type":2048,"pkt_l3_offset":14,"pkt_l4_offset":34,"pkt_len":166,"pkt_l4_len":132,"thread_ts_usec":944207338490000,"pkt":"AMCV+E3TAMCV4Bm+CABFAACYZMcAAP8RE\/OLGRYCixkWZgP\/CAEAhHghXh0LlQAAAAAAAAACAAGGowAAAAIAAAARAAAAAQAAADQ4R3XQAAAACXdlcnJtc2NoZQAAAAAAAAAAAAABAAAABQAAAAEAAAAAAAAAAgAAAAMAAAARAAAAAAAAAAAAEBCFAAAD5wAKAAAAALJaAAAAKQAKAAAAALJaAAAAKQ=="}
-01620{"flow_event_id":5,"flow_event_name":"analyse","thread_id":0,"packet_id":40,"source":"nfsv2.pcap","alias":"nDPId-test","flow_id":5,"flow_state":"finished","flow_src_packets_processed":16,"flow_dst_packets_processed":16,"flow_first_seen":944207338490000,"flow_src_last_pkt_time":944207338580000,"flow_dst_last_pkt_time":944207338580000,"flow_idle_time":200000000,"flow_src_min_l4_payload_len":124,"flow_dst_min_l4_payload_len":0,"flow_src_max_l4_payload_len":172,"flow_dst_max_l4_payload_len":128,"flow_src_tot_l4_payload_len":2168,"flow_dst_tot_l4_payload_len":1208,"midstream":0,"thread_ts_usec":944207338580000,"l3_proto":"ip4","src_ip":"139.25.22.2","dst_ip":"139.25.22.102","src_port":1023,"dst_port":2049,"l4_proto":"udp","flow_datalink":1,"flow_max_packets":3,"data_analysis": {"iat": {"min":10000,"avg":15000.0,"max":40000,"stddev":11180.3,"var":125000000.0,"ent":3.3,"data": [40000,40000,10000,10000,10000,10000,10000,10000,10000,10000,10000,10000]},"pktlen": {"min":70,"avg":147.5,"max":214,"stddev":43.1,"var":1860.8,"ent":4.9,"data": [166,138,166,90,174,70,174,70,206,170,166,138,166,138,174,170,198,138,174,170,174,70,174,70,174,170,174,70,214,70,166,138]},"bins": {"c_to_s": [0,0,0,5,9,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"s_to_c": [6,1,0,5,4,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]},"directions": [0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1]},"ndpi": {"confidence": {"6":"DPI"},"proto":"NFS","proto_id":"11","encrypted":0,"breed":"Acceptable","category_id":4,"category":"DataTransfer"}}
+02019{"flow_event_id":5,"flow_event_name":"analyse","thread_id":0,"packet_id":40,"source":"nfsv2.pcap","alias":"nDPId-test","flow_id":5,"flow_state":"finished","flow_src_packets_processed":16,"flow_dst_packets_processed":16,"flow_first_seen":944207338490000,"flow_src_last_pkt_time":944207338580000,"flow_dst_last_pkt_time":944207338580000,"flow_idle_time":200000000,"flow_src_min_l4_payload_len":124,"flow_dst_min_l4_payload_len":0,"flow_src_max_l4_payload_len":172,"flow_dst_max_l4_payload_len":128,"flow_src_tot_l4_payload_len":2168,"flow_dst_tot_l4_payload_len":1208,"midstream":0,"thread_ts_usec":944207338580000,"l3_proto":"ip4","src_ip":"139.25.22.2","dst_ip":"139.25.22.102","src_port":1023,"dst_port":2049,"l4_proto":"udp","flow_datalink":1,"flow_max_packets":3,"data_analysis": {"iat": {"min":10000,"avg":15000.0,"max":40000,"stddev":11180.3,"var":125000000.0,"ent":3.3,"data": [40000,40000,10000,10000,10000,10000,10000,10000,10000,10000,10000,10000]},"pktlen": {"min":56,"avg":133.5,"max":200,"stddev":43.1,"var":1860.8,"ent":4.9,"data": [152,124,152,76,160,56,160,56,192,156,152,124,152,124,160,156,184,124,160,156,160,56,160,56,160,156,160,56,200,56,152,124]},"bins": {"c_to_s": [0,0,0,5,9,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"s_to_c": [6,1,0,5,4,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]},"directions": [0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1],"entropies": [3.371484280,3.525987864,3.379018784,3.466069698,3.343606710,3.300534248,3.343606710,3.300534248,3.290571213,3.348722219,3.371484280,3.323238611,3.371484280,3.487642050,3.331106663,3.335901976,3.693611860,3.362390041,3.331106663,3.362183094,3.365244627,3.300534248,3.365244627,3.215625525,3.331106663,3.379842520,3.352744579,3.300534248,3.235463142,3.225106239,3.358326435,3.513812542]},"ndpi": {"confidence": {"6":"DPI"},"proto":"NFS","proto_id":"11","encrypted":0,"breed":"Acceptable","category_id":4,"category":"DataTransfer"}}
 00748{"flow_event_id":1,"flow_event_name":"new","thread_id":0,"packet_id":153,"source":"nfsv2.pcap","alias":"nDPId-test","flow_id":6,"flow_state":"info","flow_src_packets_processed":1,"flow_dst_packets_processed":0,"flow_first_seen":944207338880000,"flow_src_last_pkt_time":944207338880000,"flow_dst_last_pkt_time":944207338880000,"flow_idle_time":200000000,"flow_src_min_l4_payload_len":64,"flow_dst_min_l4_payload_len":0,"flow_src_max_l4_payload_len":64,"flow_dst_max_l4_payload_len":0,"flow_src_tot_l4_payload_len":64,"flow_dst_tot_l4_payload_len":0,"midstream":0,"thread_ts_usec":944207338880000,"l3_proto":"ip4","src_ip":"139.25.22.2","dst_ip":"139.25.22.102","src_port":3293,"dst_port":111,"l4_proto":"udp","flow_datalink":1,"flow_max_packets":3}
 00577{"packet_event_id":2,"packet_event_name":"packet-flow","thread_id":0,"packet_id":153,"source":"nfsv2.pcap","alias":"nDPId-test","flow_id":6,"flow_packet_id":1,"flow_src_last_pkt_time":944207338880000,"flow_dst_last_pkt_time":944207338880000,"flow_idle_time":200000000,"pkt_oversize":false,"pkt_caplen":106,"pkt_type":2048,"pkt_l3_offset":14,"pkt_l4_offset":34,"pkt_len":106,"pkt_l4_len":72,"thread_ts_usec":944207338880000,"pkt":"AMCV+E3TAMCV4Bm+CABFAABcZRAAAEAR0uaLGRYCixkWZgzdAG8ASKDlOErjjgAAAAAAAAACAAGGoAAAAAMAAAADAAAAAAAAAAAAAAAAAAAAAAABhqUAAAABAAAAA3VkcAAAAAAAAAAAAA=="}
 00989{"flow_event_id":7,"flow_event_name":"detected","thread_id":0,"packet_id":153,"source":"nfsv2.pcap","alias":"nDPId-test","flow_id":6,"flow_state":"info","flow_src_packets_processed":1,"flow_dst_packets_processed":0,"flow_first_seen":944207338880000,"flow_src_last_pkt_time":944207338880000,"flow_dst_last_pkt_time":944207338880000,"flow_idle_time":200000000,"flow_src_min_l4_payload_len":64,"flow_dst_min_l4_payload_len":0,"flow_src_max_l4_payload_len":64,"flow_dst_max_l4_payload_len":0,"flow_src_tot_l4_payload_len":64,"flow_dst_tot_l4_payload_len":0,"midstream":0,"thread_ts_usec":944207338880000,"l3_proto":"ip4","src_ip":"139.25.22.2","dst_ip":"139.25.22.102","src_port":3293,"dst_port":111,"l4_proto":"udp","ndpi": {"flow_risk": {"5": {"risk":"Known Proto on Non Std Port","severity":"Medium","risk_score": {"total":260,"client":230,"server":30}}},"confidence": {"6":"DPI"},"proto":"NFS","proto_id":"11","encrypted":0,"breed":"Acceptable","category_id":4,"category":"DataTransfer"}}
@@ -46,10 +46,10 @@
 ~~ total active/idle flows...: 7/7
 ~~ total timeout flows.......: 0
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-~~ total memory allocated....: 6049909 bytes
-~~ total memory freed........: 6049909 bytes
-~~ total allocations/frees...: 121703/121703
+~~ total memory allocated....: 6050861 bytes
+~~ total memory freed........: 6050861 bytes
+~~ total allocations/frees...: 121710/121710
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 ~~ json string min len.......: 490 chars
-~~ json string max len.......: 1625 chars
-~~ json string avg len.......: 1056 chars
+~~ json string max len.......: 2024 chars
+~~ json string avg len.......: 1256 chars