Added nDPIsrvd-analysed to generate CSV files from analyse events.

 * nDPIsrvd.h: iterate over JSON arrays
 * nDPId: calculate l3 payload packet entropies for analysis

Signed-off-by: Toni Uhlig <matzeton@googlemail.com>

1 files changed, 6 insertions, 6 deletions

diff --git a/test/results/git.pcap.out b/test/results/git.pcap.out
index d55aafc42..b785bc72d 100644
--- a/test/results/git.pcap.out
+++ b/test/results/git.pcap.out
@@ -5,7 +5,7 @@
 00531{"packet_event_id":2,"packet_event_name":"packet-flow","thread_id":0,"packet_id":2,"source":"git.pcap","alias":"nDPId-test","flow_id":1,"flow_packet_id":2,"flow_src_last_pkt_time":1460821630164056,"flow_dst_last_pkt_time":1460821630221958,"flow_idle_time":3285032704,"pkt_oversize":false,"pkt_caplen":74,"pkt_type":2048,"pkt_l3_offset":14,"pkt_l4_offset":34,"pkt_len":74,"pkt_l4_len":40,"thread_ts_usec":1460821630221958,"pkt":"PJcOZtCOnJcm0ghCCABFCAA8AABAAC8GnhAFmecVwKgATSTKu3dqwE5VfoYLRaASOJBfrwAAAgQFrAQCCAorjWmrAadIEgEDAwc="}
 00519{"packet_event_id":2,"packet_event_name":"packet-flow","thread_id":0,"packet_id":3,"source":"git.pcap","alias":"nDPId-test","flow_id":1,"flow_packet_id":3,"flow_src_last_pkt_time":1460821630222020,"flow_dst_last_pkt_time":1460821630221958,"flow_idle_time":3285032704,"pkt_oversize":false,"pkt_caplen":66,"pkt_type":2048,"pkt_l3_offset":14,"pkt_l4_offset":34,"pkt_len":66,"pkt_l4_len":32,"thread_ts_usec":1460821630222020,"pkt":"nJcm0ghCPJcOZtCOCABFAAA0Q1dAAEAGScnAqABNBZnnFbt3JMp+hgtFasBOVoAQAB3G2AAAAQEICgGnSCArjWmr"}
 00856{"flow_event_id":7,"flow_event_name":"detected","thread_id":0,"packet_id":4,"source":"git.pcap","alias":"nDPId-test","flow_id":1,"flow_state":"info","flow_src_packets_processed":3,"flow_dst_packets_processed":1,"flow_first_seen":1460821630164056,"flow_src_last_pkt_time":1460821630222080,"flow_dst_last_pkt_time":1460821630221958,"flow_idle_time":3285032704,"flow_src_min_l4_payload_len":0,"flow_dst_min_l4_payload_len":0,"flow_src_max_l4_payload_len":69,"flow_dst_max_l4_payload_len":0,"flow_src_tot_l4_payload_len":69,"flow_dst_tot_l4_payload_len":0,"midstream":0,"thread_ts_usec":1460821630222080,"l3_proto":"ip4","src_ip":"192.168.0.77","dst_ip":"5.153.231.21","src_port":47991,"dst_port":9418,"l4_proto":"tcp","ndpi": {"confidence": {"6":"DPI"},"proto":"Git","proto_id":"226","encrypted":0,"breed":"Safe","category_id":15,"category":"Collaborative"}}
-01708{"flow_event_id":5,"flow_event_name":"analyse","thread_id":0,"packet_id":32,"source":"git.pcap","alias":"nDPId-test","flow_id":1,"flow_state":"finished","flow_src_packets_processed":13,"flow_dst_packets_processed":19,"flow_first_seen":1460821630164056,"flow_src_last_pkt_time":1460821630544728,"flow_dst_last_pkt_time":1460821630545903,"flow_idle_time":3285032704,"flow_src_min_l4_payload_len":0,"flow_dst_min_l4_payload_len":0,"flow_src_max_l4_payload_len":527,"flow_dst_max_l4_payload_len":2880,"flow_src_tot_l4_payload_len":605,"flow_dst_tot_l4_payload_len":19825,"midstream":0,"thread_ts_usec":1460821630545903,"l3_proto":"ip4","src_ip":"192.168.0.77","dst_ip":"5.153.231.21","src_port":47991,"dst_port":9418,"l4_proto":"tcp","flow_datalink":1,"flow_max_packets":3,"data_analysis": {"iat": {"min":29,"avg":24597.4,"max":99851,"stddev":28614.0,"var":818762240.0,"ent":3.8,"data": [57902,57964,60,56073,43848,99851,54739,54730,537,49455,48900,45519,29,17836,63404,1849,203,2031,860,202,1063,209,208,710,439,1139,50571,205,50785,547,651]},"pktlen": {"min":66,"avg":704.9,"max":2946,"stddev":773.9,"var":598945.8,"ent":4.1,"data": [74,74,66,135,66,267,66,962,66,593,66,75,66,74,1506,66,1506,1506,66,1506,1506,66,2946,66,1506,1506,66,1506,1506,66,1506,1506]},"bins": {"c_to_s": [11,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"s_to_c": [5,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,11,0,1]},"directions": [0,1,0,0,1,1,0,1,0,0,1,0,1,1,1,0,1,1,0,1,1,0,1,0,1,1,0,1,1,0,1,1]},"ndpi": {"confidence": {"6":"DPI"},"proto":"Git","proto_id":"226","encrypted":0,"breed":"Safe","category_id":15,"category":"Collaborative"}}
+02107{"flow_event_id":5,"flow_event_name":"analyse","thread_id":0,"packet_id":32,"source":"git.pcap","alias":"nDPId-test","flow_id":1,"flow_state":"finished","flow_src_packets_processed":13,"flow_dst_packets_processed":19,"flow_first_seen":1460821630164056,"flow_src_last_pkt_time":1460821630544728,"flow_dst_last_pkt_time":1460821630545903,"flow_idle_time":3285032704,"flow_src_min_l4_payload_len":0,"flow_dst_min_l4_payload_len":0,"flow_src_max_l4_payload_len":527,"flow_dst_max_l4_payload_len":2880,"flow_src_tot_l4_payload_len":605,"flow_dst_tot_l4_payload_len":19825,"midstream":0,"thread_ts_usec":1460821630545903,"l3_proto":"ip4","src_ip":"192.168.0.77","dst_ip":"5.153.231.21","src_port":47991,"dst_port":9418,"l4_proto":"tcp","flow_datalink":1,"flow_max_packets":3,"data_analysis": {"iat": {"min":29,"avg":24597.4,"max":99851,"stddev":28614.0,"var":818762240.0,"ent":3.8,"data": [57902,57964,60,56073,43848,99851,54739,54730,537,49455,48900,45519,29,17836,63404,1849,203,2031,860,202,1063,209,208,710,439,1139,50571,205,50785,547,651]},"pktlen": {"min":52,"avg":690.9,"max":2932,"stddev":773.9,"var":598945.8,"ent":4.1,"data": [60,60,52,121,52,253,52,948,52,579,52,61,52,60,1492,52,1492,1492,52,1492,1492,52,2932,52,1492,1492,52,1492,1492,52,1492,1492]},"bins": {"c_to_s": [11,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"s_to_c": [5,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,11,0,1]},"directions": [0,1,0,0,1,1,0,1,0,0,1,0,1,1,1,0,1,1,0,1,1,0,1,0,1,1,0,1,1,0,1,1],"entropies": [4.739262104,5.279368877,5.115703106,5.628006458,5.195351124,5.731617451,5.115702629,4.962421417,5.154164791,5.045848370,5.195351601,5.288749218,5.233812809,5.389901161,4.890160084,5.154164791,6.262699604,7.849300385,5.154164791,7.861139297,7.866855145,5.154164791,7.887691021,5.024262905,7.851975918,7.853373528,5.154164791,7.871936798,7.800623894,5.115703106,7.834641933,7.837094784]},"ndpi": {"confidence": {"6":"DPI"},"proto":"Git","proto_id":"226","encrypted":0,"breed":"Safe","category_id":15,"category":"Collaborative"}}
 00906{"flow_event_id":2,"flow_event_name":"end","thread_id":0,"packet_id":90,"source":"git.pcap","alias":"nDPId-test","flow_id":1,"flow_state":"finished","flow_src_packets_processed":41,"flow_dst_packets_processed":49,"flow_first_seen":1460821630164056,"flow_src_last_pkt_time":1460821631220936,"flow_dst_last_pkt_time":1460821631269756,"flow_idle_time":3285032704,"flow_src_min_l4_payload_len":0,"flow_dst_min_l4_payload_len":0,"flow_src_max_l4_payload_len":527,"flow_dst_max_l4_payload_len":2880,"flow_src_tot_l4_payload_len":605,"flow_dst_tot_l4_payload_len":67444,"midstream":0,"thread_ts_usec":1460821631269756,"l3_proto":"ip4","src_ip":"192.168.0.77","dst_ip":"5.153.231.21","src_port":47991,"dst_port":9418,"l4_proto":"tcp","flow_datalink":1,"flow_max_packets":3,"ndpi": {"confidence": {"6":"DPI"},"proto":"Git","proto_id":"226","encrypted":0,"breed":"Safe","category_id":15,"category":"Collaborative"}}
 00556{"daemon_event_id":3,"daemon_event_name":"shutdown","thread_id":0,"packet_id":90,"source":"git.pcap","alias":"nDPId-test","packets-captured":90,"packets-processed":90,"total-skipped-flows":0,"total-l4-payload-len":68049,"total-not-detected-flows":0,"total-guessed-flows":0,"total-detected-flows":1,"total-detection-updates":0,"total-updates":0,"current-active-flows":0,"total-active-flows":1,"total-idle-flows":1,"total-compressions":0,"total-compression-diff":0,"current-compression-diff":0,"total-events-serialized":10,"global_ts_usec":1460821631269756}
 ~~~~~~~~~~~~~~~~~~~~ SUMMARY ~~~~~~~~~~~~~~~~~~~~
@@ -16,10 +16,10 @@
 ~~ total active/idle flows...: 1/1
 ~~ total timeout flows.......: 0
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-~~ total memory allocated....: 6038251 bytes
-~~ total memory freed........: 6038251 bytes
-~~ total allocations/frees...: 121577/121577
+~~ total memory allocated....: 6038387 bytes
+~~ total memory freed........: 6038387 bytes
+~~ total allocations/frees...: 121578/121578
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 ~~ json string min len.......: 488 chars
-~~ json string max len.......: 1713 chars
-~~ json string avg len.......: 1042 chars
+~~ json string max len.......: 2112 chars
+~~ json string avg len.......: 1217 chars