Added nDPIsrvd-analysed to generate CSV files from analyse events.

 * nDPIsrvd.h: iterate over JSON arrays
 * nDPId: calculate l3 payload packet entropies for analysis

Signed-off-by: Toni Uhlig <matzeton@googlemail.com>

1 files changed, 6 insertions, 6 deletions

diff --git a/test/results/ssh.pcap.out b/test/results/ssh.pcap.out
index c7a0e9767..da6507bbe 100644
--- a/test/results/ssh.pcap.out
+++ b/test/results/ssh.pcap.out
@@ -9,7 +9,7 @@
 01252{"flow_event_id":8,"flow_event_name":"detection-update","thread_id":0,"packet_id":6,"source":"ssh.pcap","alias":"nDPId-test","flow_id":1,"flow_state":"info","flow_src_packets_processed":4,"flow_dst_packets_processed":2,"flow_first_seen":1320435464760244,"flow_src_last_pkt_time":1320435464768726,"flow_dst_last_pkt_time":1320435464768382,"flow_idle_time":3285032704,"flow_src_min_l4_payload_len":0,"flow_dst_min_l4_payload_len":0,"flow_src_max_l4_payload_len":21,"flow_dst_max_l4_payload_len":21,"flow_src_tot_l4_payload_len":21,"flow_dst_tot_l4_payload_len":21,"midstream":0,"thread_ts_usec":1320435464768726,"l3_proto":"ip4","src_ip":"172.16.238.1","dst_ip":"172.16.238.168","src_port":58395,"dst_port":22,"l4_proto":"tcp","ndpi": {"flow_risk": {"18": {"risk":"SSH Obsolete Cli Vers\/Cipher","severity":"High","risk_score": {"total":500,"client":350,"server":150}},"19": {"risk":"SSH Obsolete Ser Vers\/Cipher","severity":"Medium","risk_score": {"total":510,"client":355,"server":155}}},"confidence": {"6":"DPI"},"proto":"SSH","proto_id":"92","encrypted":1,"breed":"Acceptable","category_id":12,"category":"RemoteAccess","ssh": {"client_signature":"SSH-2.0-OpenSSH_5.3","server_signature":"SSH-2.0-OpenSSH_5.6","hassh_client":"","hassh_server":""}}}
 01286{"flow_event_id":8,"flow_event_name":"detection-update","thread_id":0,"packet_id":8,"source":"ssh.pcap","alias":"nDPId-test","flow_id":1,"flow_state":"info","flow_src_packets_processed":5,"flow_dst_packets_processed":3,"flow_first_seen":1320435464760244,"flow_src_last_pkt_time":1320435464769196,"flow_dst_last_pkt_time":1320435464769170,"flow_idle_time":3285032704,"flow_src_min_l4_payload_len":0,"flow_dst_min_l4_payload_len":0,"flow_src_max_l4_payload_len":904,"flow_dst_max_l4_payload_len":21,"flow_src_tot_l4_payload_len":925,"flow_dst_tot_l4_payload_len":21,"midstream":0,"thread_ts_usec":1320435464769196,"l3_proto":"ip4","src_ip":"172.16.238.1","dst_ip":"172.16.238.168","src_port":58395,"dst_port":22,"l4_proto":"tcp","ndpi": {"flow_risk": {"18": {"risk":"SSH Obsolete Cli Vers\/Cipher","severity":"High","risk_score": {"total":500,"client":350,"server":150}},"19": {"risk":"SSH Obsolete Ser Vers\/Cipher","severity":"Medium","risk_score": {"total":510,"client":355,"server":155}}},"confidence": {"6":"DPI"},"proto":"SSH","proto_id":"92","encrypted":1,"breed":"Acceptable","category_id":12,"category":"RemoteAccess","ssh": {"client_signature":"SSH-2.0-OpenSSH_5.3","server_signature":"SSH-2.0-OpenSSH_5.6","hassh_client":"21B457A327CE7A2D4FCE5EF2C42400BD","hassh_server":""}}}
 01321{"flow_event_id":8,"flow_event_name":"detection-update","thread_id":0,"packet_id":10,"source":"ssh.pcap","alias":"nDPId-test","flow_id":1,"flow_state":"info","flow_src_packets_processed":5,"flow_dst_packets_processed":5,"flow_first_seen":1320435464760244,"flow_src_last_pkt_time":1320435464769196,"flow_dst_last_pkt_time":1320435464770779,"flow_idle_time":3285032704,"flow_src_min_l4_payload_len":0,"flow_dst_min_l4_payload_len":0,"flow_src_max_l4_payload_len":904,"flow_dst_max_l4_payload_len":784,"flow_src_tot_l4_payload_len":925,"flow_dst_tot_l4_payload_len":805,"midstream":0,"thread_ts_usec":1320435464770779,"l3_proto":"ip4","src_ip":"172.16.238.1","dst_ip":"172.16.238.168","src_port":58395,"dst_port":22,"l4_proto":"tcp","ndpi": {"flow_risk": {"18": {"risk":"SSH Obsolete Cli Vers\/Cipher","severity":"High","risk_score": {"total":500,"client":350,"server":150}},"19": {"risk":"SSH Obsolete Ser Vers\/Cipher","severity":"Medium","risk_score": {"total":510,"client":355,"server":155}}},"confidence": {"6":"DPI"},"proto":"SSH","proto_id":"92","encrypted":1,"breed":"Acceptable","category_id":12,"category":"RemoteAccess","ssh": {"client_signature":"SSH-2.0-OpenSSH_5.3","server_signature":"SSH-2.0-OpenSSH_5.6","hassh_client":"21B457A327CE7A2D4FCE5EF2C42400BD","hassh_server":"B1C6C0D56317555B85C7005A3DE29325"}}}
-01964{"flow_event_id":5,"flow_event_name":"analyse","thread_id":0,"packet_id":32,"source":"ssh.pcap","alias":"nDPId-test","flow_id":1,"flow_state":"finished","flow_src_packets_processed":18,"flow_dst_packets_processed":14,"flow_first_seen":1320435464760244,"flow_src_last_pkt_time":1320435472330349,"flow_dst_last_pkt_time":1320435469423179,"flow_idle_time":3285032704,"flow_src_min_l4_payload_len":0,"flow_dst_min_l4_payload_len":0,"flow_src_max_l4_payload_len":904,"flow_dst_max_l4_payload_len":784,"flow_src_tot_l4_payload_len":1509,"flow_dst_tot_l4_payload_len":1885,"midstream":0,"thread_ts_usec":1320435472330349,"l3_proto":"ip4","src_ip":"172.16.238.1","dst_ip":"172.16.238.168","src_port":58395,"dst_port":22,"l4_proto":"tcp","flow_datalink":1,"flow_max_packets":3,"data_analysis": {"iat": {"min":26,"avg":394614.2,"max":2907110,"stddev":888738.9,"var":789856780288.0,"ent":2.5,"data": [26,41,8112,8146,295,788,470,140,1469,1611,306,1791,1560,1614,14729,13069,1842,42337,40496,170,257,393,251,40593,51194,91555,2632288,2632557,1868772,1869058,2907110]},"pktlen": {"min":66,"avg":172.7,"max":970,"stddev":230.1,"var":52961.8,"ent":4.2,"data": [78,74,66,87,66,87,66,970,66,850,66,90,218,66,210,786,66,82,66,114,66,114,66,130,66,146,66,210,66,146,66,210]},"bins": {"c_to_s": [12,1,1,0,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"s_to_c": [8,1,2,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]},"directions": [0,1,0,1,0,0,1,0,1,1,0,0,1,0,0,1,0,0,1,0,1,1,0,0,1,1,0,0,1,1,0,0]},"ndpi": {"flow_risk": {"18": {"risk":"SSH Obsolete Cli Vers\/Cipher","severity":"High","risk_score": {"total":500,"client":350,"server":150}},"19": {"risk":"SSH Obsolete Ser Vers\/Cipher","severity":"Medium","risk_score": {"total":510,"client":355,"server":155}}},"confidence": {"6":"DPI"},"proto":"SSH","proto_id":"92","encrypted":1,"breed":"Acceptable","category_id":12,"category":"RemoteAccess"}}
+02363{"flow_event_id":5,"flow_event_name":"analyse","thread_id":0,"packet_id":32,"source":"ssh.pcap","alias":"nDPId-test","flow_id":1,"flow_state":"finished","flow_src_packets_processed":18,"flow_dst_packets_processed":14,"flow_first_seen":1320435464760244,"flow_src_last_pkt_time":1320435472330349,"flow_dst_last_pkt_time":1320435469423179,"flow_idle_time":3285032704,"flow_src_min_l4_payload_len":0,"flow_dst_min_l4_payload_len":0,"flow_src_max_l4_payload_len":904,"flow_dst_max_l4_payload_len":784,"flow_src_tot_l4_payload_len":1509,"flow_dst_tot_l4_payload_len":1885,"midstream":0,"thread_ts_usec":1320435472330349,"l3_proto":"ip4","src_ip":"172.16.238.1","dst_ip":"172.16.238.168","src_port":58395,"dst_port":22,"l4_proto":"tcp","flow_datalink":1,"flow_max_packets":3,"data_analysis": {"iat": {"min":26,"avg":394614.2,"max":2907110,"stddev":888738.9,"var":789856780288.0,"ent":2.5,"data": [26,41,8112,8146,295,788,470,140,1469,1611,306,1791,1560,1614,14729,13069,1842,42337,40496,170,257,393,251,40593,51194,91555,2632288,2632557,1868772,1869058,2907110]},"pktlen": {"min":52,"avg":158.7,"max":956,"stddev":230.1,"var":52961.8,"ent":4.1,"data": [64,60,52,73,52,73,52,956,52,836,52,76,204,52,196,772,52,68,52,100,52,100,52,116,52,132,52,196,52,132,52,196]},"bins": {"c_to_s": [12,1,1,0,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"s_to_c": [8,1,2,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]},"directions": [0,1,0,1,0,0,1,0,1,1,0,0,1,0,0,1,0,0,1,0,1,1,0,0,1,1,0,0,1,1,0,0],"entropies": [4.495864868,5.031404495,4.947339535,5.395304680,4.870416641,5.379396915,4.940637589,5.147055149,4.940637589,5.183596134,4.923395157,4.404554367,6.511710644,4.985801220,6.696379662,7.508841991,4.884933472,4.511087418,4.815073490,5.981212139,4.902175903,6.028761387,4.894361019,6.251031399,4.940637589,6.350845814,4.932822704,6.810175419,4.853535175,6.303876877,4.902175426,6.814750671]},"ndpi": {"flow_risk": {"18": {"risk":"SSH Obsolete Cli Vers\/Cipher","severity":"High","risk_score": {"total":500,"client":350,"server":150}},"19": {"risk":"SSH Obsolete Ser Vers\/Cipher","severity":"Medium","risk_score": {"total":510,"client":355,"server":155}}},"confidence": {"6":"DPI"},"proto":"SSH","proto_id":"92","encrypted":1,"breed":"Acceptable","category_id":12,"category":"RemoteAccess"}}
 01168{"flow_event_id":2,"flow_event_name":"end","thread_id":0,"packet_id":258,"source":"ssh.pcap","alias":"nDPId-test","flow_id":1,"flow_state":"finished","flow_src_packets_processed":159,"flow_dst_packets_processed":99,"flow_first_seen":1320435464760244,"flow_src_last_pkt_time":1320435713237065,"flow_dst_last_pkt_time":1320435713237024,"flow_idle_time":3285032704,"flow_src_min_l4_payload_len":0,"flow_dst_min_l4_payload_len":0,"flow_src_max_l4_payload_len":904,"flow_dst_max_l4_payload_len":1280,"flow_src_tot_l4_payload_len":5109,"flow_dst_tot_l4_payload_len":13389,"midstream":0,"thread_ts_usec":1320435713237065,"l3_proto":"ip4","src_ip":"172.16.238.1","dst_ip":"172.16.238.168","src_port":58395,"dst_port":22,"l4_proto":"tcp","flow_datalink":1,"flow_max_packets":3,"ndpi": {"flow_risk": {"18": {"risk":"SSH Obsolete Cli Vers\/Cipher","severity":"High","risk_score": {"total":500,"client":350,"server":150}},"19": {"risk":"SSH Obsolete Ser Vers\/Cipher","severity":"Medium","risk_score": {"total":510,"client":355,"server":155}}},"confidence": {"6":"DPI"},"proto":"SSH","proto_id":"92","encrypted":1,"breed":"Acceptable","category_id":12,"category":"RemoteAccess"}}
 00559{"daemon_event_id":3,"daemon_event_name":"shutdown","thread_id":0,"packet_id":258,"source":"ssh.pcap","alias":"nDPId-test","packets-captured":258,"packets-processed":258,"total-skipped-flows":0,"total-l4-payload-len":18498,"total-not-detected-flows":0,"total-guessed-flows":0,"total-detected-flows":1,"total-detection-updates":4,"total-updates":0,"current-active-flows":0,"total-active-flows":1,"total-idle-flows":1,"total-compressions":0,"total-compression-diff":0,"current-compression-diff":0,"total-events-serialized":14,"global_ts_usec":1320435713237065}
 ~~~~~~~~~~~~~~~~~~~~ SUMMARY ~~~~~~~~~~~~~~~~~~~~
@@ -20,10 +20,10 @@
 ~~ total active/idle flows...: 1/1
 ~~ total timeout flows.......: 0
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-~~ total memory allocated....: 6045127 bytes
-~~ total memory freed........: 6045127 bytes
-~~ total allocations/frees...: 121749/121749
+~~ total memory allocated....: 6045263 bytes
+~~ total memory freed........: 6045263 bytes
+~~ total allocations/frees...: 121750/121750
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 ~~ json string min len.......: 488 chars
-~~ json string max len.......: 1969 chars
-~~ json string avg len.......: 1184 chars
+~~ json string max len.......: 2368 chars
+~~ json string avg len.......: 1359 chars