Added nDPIsrvd-analysed to generate CSV files from analyse events.

 * nDPIsrvd.h: iterate over JSON arrays
 * nDPId: calculate l3 payload packet entropies for analysis

Signed-off-by: Toni Uhlig <matzeton@googlemail.com>

1 files changed, 6 insertions, 6 deletions

diff --git a/test/results/6in4tunnel.pcap.out b/test/results/6in4tunnel.pcap.out
index 67bff0c25..97ef1b782 100644
--- a/test/results/6in4tunnel.pcap.out
+++ b/test/results/6in4tunnel.pcap.out
@@ -4,7 +4,7 @@
 00628{"packet_event_id":2,"packet_event_name":"packet-flow","thread_id":0,"packet_id":1,"source":"6in4tunnel.pcap","alias":"nDPId-test","flow_id":1,"flow_packet_id":1,"flow_src_last_pkt_time":1444236893450580,"flow_dst_last_pkt_time":1444236893450580,"flow_idle_time":620000000,"pkt_oversize":false,"pkt_caplen":138,"pkt_type":2048,"pkt_l3_offset":14,"pkt_l4_offset":34,"pkt_len":138,"pkt_l4_len":104,"thread_ts_usec":1444236893450580,"pkt":"ACKQ3jvZAAAkzoE0CABFAAB8tYFAAP8pFzeuA0kYuGn\/GmAAAAAAQDo\/IAEEcB8XAT8+lw7\/\/nNN7CYEqIAAAQAgAAAAAAIksAGAAOC9XY8BWl1OFVYAAAAAqN0GAAAAAAAQERITFBUWFxgZGhscHR4fICEiIyQlJicoKSorLC0uLzAxMjM0NTY3"}
 00627{"packet_event_id":2,"packet_event_name":"packet-flow","thread_id":0,"packet_id":2,"source":"6in4tunnel.pcap","alias":"nDPId-test","flow_id":1,"flow_packet_id":2,"flow_src_last_pkt_time":1444236893450580,"flow_dst_last_pkt_time":1444236893555356,"flow_idle_time":620000000,"pkt_oversize":false,"pkt_caplen":138,"pkt_type":2048,"pkt_l3_offset":14,"pkt_l4_offset":34,"pkt_len":138,"pkt_l4_len":104,"thread_ts_usec":1444236893555356,"pkt":"AAAkzoE0ACKQ3jvZCABFAAB8xlZAAPgpDWK4af8argNJGGAAAAAAQDo3JgSogAABACAAAAAAAiSwASABBHAfFwE\/PpcO\/\/5zTeyBAN+9XY8BWl1OFVYAAAAAqN0GAAAAAAAQERITFBUWFxgZGhscHR4fICEiIyQlJicoKSorLC0uLzAxMjM0NTY3"}
 00711{"packet_event_id":2,"packet_event_name":"packet-flow","thread_id":0,"packet_id":3,"source":"6in4tunnel.pcap","alias":"nDPId-test","flow_id":1,"flow_packet_id":3,"flow_src_last_pkt_time":1444236894230722,"flow_dst_last_pkt_time":1444236893555356,"flow_idle_time":620000000,"pkt_oversize":false,"pkt_caplen":200,"pkt_type":2048,"pkt_l3_offset":14,"pkt_l4_offset":34,"pkt_len":200,"pkt_l4_len":166,"thread_ts_usec":1444236894230722,"pkt":"ACKQ3jvZAAAkzoE0CABFAAC6tdFAAP8pFqmuA0kYuGn\/GmAAAAAAfjpAIAEEcB8WAT8AAAAAAAAAAiYEqIAAAQAgAAAAAAIksAEBA9KAAAAAAGAAAAAATgY2JgSogAABACAAAAAAAiSwASABBHAfFwE\/JaMykhb5LOAD4exLUvt9fRlwFpiAGABJEPkAAAEBCAq0MT0ACHX6xhcDAwApoxPniAjxmmXGKxqxVV6nOvla9FPS7Dtl2rRDlmVhpOKK9OFyB\/XihP8="}
-01600{"flow_event_id":5,"flow_event_name":"analyse","thread_id":0,"packet_id":32,"source":"6in4tunnel.pcap","alias":"nDPId-test","flow_id":1,"flow_state":"info","flow_src_packets_processed":18,"flow_dst_packets_processed":14,"flow_first_seen":1444236893450580,"flow_src_last_pkt_time":1444236901127917,"flow_dst_last_pkt_time":1444236901118187,"flow_idle_time":620000000,"flow_src_min_l4_payload_len":72,"flow_dst_min_l4_payload_len":0,"flow_src_max_l4_payload_len":276,"flow_dst_max_l4_payload_len":1877,"flow_src_tot_l4_payload_len":2127,"flow_dst_tot_l4_payload_len":4797,"midstream":0,"thread_ts_usec":1444236901127917,"l3_proto":"ip4","src_ip":"174.3.73.24","dst_ip":"184.105.255.26","l4_proto":41,"flow_datalink":1,"flow_max_packets":3,"data_analysis": {"iat": {"min":105,"avg":494998.2,"max":1005120,"stddev":454962.0,"var":206990442496.0,"ent":4.2,"data": [104776,780142,221063,1000457,1001744,1001146,1001712,1005120,1001052,1000771,1001064,1001072,1001370,999940,1001888,1003131,365420,1118,348987,4072,96728,99146,95730,758,97863,1021,105,98080,140,8789,539]},"pktlen": {"min":106,"avg":250.4,"max":1911,"stddev":383.0,"var":146712.7,"ent":4.2,"data": [138,138,200,138,138,138,138,138,138,138,138,138,138,138,138,138,138,133,133,273,261,114,114,106,310,106,1504,1911,106,106,268,159]},"bins": {"c_to_s": [0,0,4,11,0,1,0,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"s_to_c": [0,0,2,8,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,1]},"directions": [0,1,0,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,0,1,1,0,1,0,0,1,1,1,0,0,0,0]}}
+01994{"flow_event_id":5,"flow_event_name":"analyse","thread_id":0,"packet_id":32,"source":"6in4tunnel.pcap","alias":"nDPId-test","flow_id":1,"flow_state":"info","flow_src_packets_processed":18,"flow_dst_packets_processed":14,"flow_first_seen":1444236893450580,"flow_src_last_pkt_time":1444236901127917,"flow_dst_last_pkt_time":1444236901118187,"flow_idle_time":620000000,"flow_src_min_l4_payload_len":72,"flow_dst_min_l4_payload_len":0,"flow_src_max_l4_payload_len":276,"flow_dst_max_l4_payload_len":1877,"flow_src_tot_l4_payload_len":2127,"flow_dst_tot_l4_payload_len":4797,"midstream":0,"thread_ts_usec":1444236901127917,"l3_proto":"ip4","src_ip":"174.3.73.24","dst_ip":"184.105.255.26","l4_proto":41,"flow_datalink":1,"flow_max_packets":3,"data_analysis": {"iat": {"min":105,"avg":494998.2,"max":1005120,"stddev":454962.0,"var":206990442496.0,"ent":4.2,"data": [104776,780142,221063,1000457,1001744,1001146,1001712,1005120,1001052,1000771,1001064,1001072,1001370,999940,1001888,1003131,365420,1118,348987,4072,96728,99146,95730,758,97863,1021,105,98080,140,8789,539]},"pktlen": {"min":92,"avg":236.4,"max":1897,"stddev":383.0,"var":146712.7,"ent":4.1,"data": [124,124,186,124,124,124,124,124,124,124,124,124,124,124,124,124,124,119,119,259,247,100,100,92,296,92,1490,1897,92,92,254,145]},"bins": {"c_to_s": [0,0,4,11,0,1,0,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"s_to_c": [0,0,2,8,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,1]},"directions": [0,1,0,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,0,1,1,0,1,0,0,1,1,1,0,0,0,0],"entropies": [5.680680275,5.741242886,5.591180325,5.686768055,5.741242886,5.686768055,5.741242886,5.664551258,5.741242886,5.729067326,5.773500919,5.648445129,5.741242886,5.664551258,5.725113869,5.680680275,5.735155106,4.719979763,4.710355759,4.773607731,4.870984077,5.180728912,5.772128105,5.515571117,5.818006039,5.609004974,6.932967663,6.965810776,5.515571117,5.514929771,6.708754063,6.001224995]}}
 00779{"flow_event_id":9,"flow_event_name":"not-detected","thread_id":0,"packet_id":32,"source":"6in4tunnel.pcap","alias":"nDPId-test","flow_id":1,"flow_state":"info","flow_src_packets_processed":18,"flow_dst_packets_processed":14,"flow_first_seen":1444236893450580,"flow_src_last_pkt_time":1444236901127917,"flow_dst_last_pkt_time":1444236901118187,"flow_idle_time":620000000,"flow_src_min_l4_payload_len":72,"flow_dst_min_l4_payload_len":0,"flow_src_max_l4_payload_len":276,"flow_dst_max_l4_payload_len":1877,"flow_src_tot_l4_payload_len":2127,"flow_dst_tot_l4_payload_len":4797,"midstream":0,"thread_ts_usec":1444236901127917,"l3_proto":"ip4","src_ip":"174.3.73.24","dst_ip":"184.105.255.26","l4_proto":41,"ndpi": {"proto":"Unknown","proto_id":"0","encrypted":0,"breed":"Unrated"}}
 00818{"flow_event_id":3,"flow_event_name":"idle","thread_id":0,"packet_id":127,"source":"6in4tunnel.pcap","alias":"nDPId-test","flow_id":1,"flow_state":"finished","flow_src_packets_processed":66,"flow_dst_packets_processed":61,"flow_first_seen":1444236893450580,"flow_src_last_pkt_time":1444236915478638,"flow_dst_last_pkt_time":1444236915586195,"flow_idle_time":620000000,"flow_src_min_l4_payload_len":72,"flow_dst_min_l4_payload_len":0,"flow_src_max_l4_payload_len":1470,"flow_dst_max_l4_payload_len":1877,"flow_src_tot_l4_payload_len":11600,"flow_dst_tot_l4_payload_len":24375,"midstream":0,"thread_ts_usec":1444236915586195,"l3_proto":"ip4","src_ip":"174.3.73.24","dst_ip":"184.105.255.26","l4_proto":41,"flow_datalink":1,"flow_max_packets":3,"ndpi": {"proto":"Unknown","proto_id":"0","encrypted":0,"breed":"Unrated"}}
 00566{"daemon_event_id":3,"daemon_event_name":"shutdown","thread_id":0,"packet_id":127,"source":"6in4tunnel.pcap","alias":"nDPId-test","packets-captured":127,"packets-processed":127,"total-skipped-flows":0,"total-l4-payload-len":35975,"total-not-detected-flows":1,"total-guessed-flows":0,"total-detected-flows":0,"total-detection-updates":0,"total-updates":0,"current-active-flows":0,"total-active-flows":1,"total-idle-flows":1,"total-compressions":0,"total-compression-diff":0,"current-compression-diff":0,"total-events-serialized":10,"global_ts_usec":1444236915586195}
@@ -16,10 +16,10 @@
 ~~ total active/idle flows...: 1/1
 ~~ total timeout flows.......: 0
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-~~ total memory allocated....: 6039324 bytes
-~~ total memory freed........: 6039324 bytes
-~~ total allocations/frees...: 121614/121614
+~~ total memory allocated....: 6039460 bytes
+~~ total memory freed........: 6039460 bytes
+~~ total allocations/frees...: 121615/121615
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 ~~ json string min len.......: 495 chars
-~~ json string max len.......: 1605 chars
-~~ json string avg len.......: 1021 chars
+~~ json string max len.......: 1999 chars
+~~ json string avg len.......: 1206 chars