Added nDPIsrvd-analysed to generate CSV files from analyse events.

 * nDPIsrvd.h: iterate over JSON arrays
 * nDPId: calculate l3 payload packet entropies for analysis

Signed-off-by: Toni Uhlig <matzeton@googlemail.com>

1 files changed, 6 insertions, 6 deletions

diff --git a/test/results/icmp-tunnel.pcap.out b/test/results/icmp-tunnel.pcap.out
index a3263d62d..7bc547eb3 100644
--- a/test/results/icmp-tunnel.pcap.out
+++ b/test/results/icmp-tunnel.pcap.out
@@ -5,7 +5,7 @@
 00977{"flow_event_id":7,"flow_event_name":"detected","thread_id":0,"packet_id":3,"source":"icmp-tunnel.pcap","alias":"nDPId-test","flow_id":1,"flow_state":"info","flow_src_packets_processed":1,"flow_dst_packets_processed":0,"flow_first_seen":1360227866459330,"flow_src_last_pkt_time":1360227866459330,"flow_dst_last_pkt_time":1360227866459330,"flow_idle_time":140000000,"flow_src_min_l4_payload_len":92,"flow_dst_min_l4_payload_len":0,"flow_src_max_l4_payload_len":92,"flow_dst_max_l4_payload_len":0,"flow_src_tot_l4_payload_len":92,"flow_dst_tot_l4_payload_len":0,"midstream":0,"thread_ts_usec":1360227866459330,"l3_proto":"ip4","src_ip":"192.168.154.131","dst_ip":"192.168.154.132","l4_proto":"icmp","ndpi": {"flow_risk": {"17": {"risk":"Malformed Packet","severity":"Low","risk_score": {"total":260,"client":130,"server":130}}},"confidence": {"6":"DPI"},"proto":"ICMP","proto_id":"81","encrypted":0,"breed":"Acceptable","category_id":14,"category":"Network","entropy":5.703333}}
 00609{"packet_event_id":2,"packet_event_name":"packet-flow","thread_id":0,"packet_id":4,"source":"icmp-tunnel.pcap","alias":"nDPId-test","flow_id":1,"flow_packet_id":2,"flow_src_last_pkt_time":1360227867458100,"flow_dst_last_pkt_time":1360227866459330,"flow_idle_time":140000000,"pkt_oversize":false,"pkt_caplen":126,"pkt_type":2048,"pkt_l3_offset":14,"pkt_l4_offset":34,"pkt_len":126,"pkt_l4_len":92,"thread_ts_usec":1360227867458100,"pkt":"AAwpy+OCAAwpzwzBCABFAABwAABAAEABhDTAqJqDwKiahAgAAAD+\/wAARQAAVAAAQABAASPpCl8BAQpfAQIIAH3tPQgAAi5uE1EKRgYACAkKCwwNDg8QERITFBUWFxgZGhscHR4fICEiIyQlJicoKSorLC0uLzAxMjM0NTY3"}
 00610{"packet_event_id":2,"packet_event_name":"packet-flow","thread_id":0,"packet_id":5,"source":"icmp-tunnel.pcap","alias":"nDPId-test","flow_id":1,"flow_packet_id":3,"flow_src_last_pkt_time":1360227868458136,"flow_dst_last_pkt_time":1360227866459330,"flow_idle_time":140000000,"pkt_oversize":false,"pkt_caplen":126,"pkt_type":2048,"pkt_l3_offset":14,"pkt_l4_offset":34,"pkt_len":126,"pkt_l4_len":92,"thread_ts_usec":1360227868458136,"pkt":"AAwpy+OCAAwpzwzBCABFAABwAABAAEABhDTAqJqDwKiahAgAAAD+\/wAARQAAVAAAQABAASPpCl8BAQpfAQIIAD\/sPQgAAy9uE1FHRgYACAkKCwwNDg8QERITFBUWFxgZGhscHR4fICEiIyQlJicoKSorLC0uLzAxMjM0NTY3"}
-01906{"flow_event_id":5,"flow_event_name":"analyse","thread_id":0,"packet_id":36,"source":"icmp-tunnel.pcap","alias":"nDPId-test","flow_id":1,"flow_state":"finished","flow_src_packets_processed":23,"flow_dst_packets_processed":9,"flow_first_seen":1360227866459330,"flow_src_last_pkt_time":1360227888466859,"flow_dst_last_pkt_time":1360227888466987,"flow_idle_time":140000000,"flow_src_min_l4_payload_len":92,"flow_dst_min_l4_payload_len":0,"flow_src_max_l4_payload_len":92,"flow_dst_max_l4_payload_len":92,"flow_src_tot_l4_payload_len":2116,"flow_dst_tot_l4_payload_len":828,"midstream":0,"thread_ts_usec":1360227888466987,"l3_proto":"ip4","src_ip":"192.168.154.131","dst_ip":"192.168.154.132","l4_proto":"icmp","flow_datalink":1,"flow_max_packets":3,"data_analysis": {"iat": {"min":998770,"avg":1419844.6,"max":13999352,"stddev":2296693.5,"var":5274800750592.0,"ent":4.2,"data": [998770,1000036,1000056,999983,1000051,1000074,1000009,1000032,1000047,1000127,999991,999982,1000043,999922,13999352,1001250,1001214,1000977,1001002,1001107,1001081,1000973,1000923,1000944,1000921,1001115,1001144,1001036,1001015,1001004,1001005]},"pktlen": {"min":126,"avg":126.0,"max":126,"stddev":0.0,"var":0.0,"ent":5.0,"data": [126,126,126,126,126,126,126,126,126,126,126,126,126,126,126,126,126,126,126,126,126,126,126,126,126,126,126,126,126,126,126,126]},"bins": {"c_to_s": [0,0,23,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"s_to_c": [0,0,9,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]},"directions": [0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1]},"ndpi": {"flow_risk": {"17": {"risk":"Malformed Packet","severity":"Low","risk_score": {"total":260,"client":130,"server":130}}},"confidence": {"6":"DPI"},"proto":"ICMP","proto_id":"81","encrypted":0,"breed":"Acceptable","category_id":14,"category":"Network"}}
+02305{"flow_event_id":5,"flow_event_name":"analyse","thread_id":0,"packet_id":36,"source":"icmp-tunnel.pcap","alias":"nDPId-test","flow_id":1,"flow_state":"finished","flow_src_packets_processed":23,"flow_dst_packets_processed":9,"flow_first_seen":1360227866459330,"flow_src_last_pkt_time":1360227888466859,"flow_dst_last_pkt_time":1360227888466987,"flow_idle_time":140000000,"flow_src_min_l4_payload_len":92,"flow_dst_min_l4_payload_len":0,"flow_src_max_l4_payload_len":92,"flow_dst_max_l4_payload_len":92,"flow_src_tot_l4_payload_len":2116,"flow_dst_tot_l4_payload_len":828,"midstream":0,"thread_ts_usec":1360227888466987,"l3_proto":"ip4","src_ip":"192.168.154.131","dst_ip":"192.168.154.132","l4_proto":"icmp","flow_datalink":1,"flow_max_packets":3,"data_analysis": {"iat": {"min":998770,"avg":1419844.6,"max":13999352,"stddev":2296693.5,"var":5274800750592.0,"ent":4.2,"data": [998770,1000036,1000056,999983,1000051,1000074,1000009,1000032,1000047,1000127,999991,999982,1000043,999922,13999352,1001250,1001214,1000977,1001002,1001107,1001081,1000973,1000923,1000944,1000921,1001115,1001144,1001036,1001015,1001004,1001005]},"pktlen": {"min":112,"avg":112.0,"max":112,"stddev":0.0,"var":0.0,"ent":5.0,"data": [112,112,112,112,112,112,112,112,112,112,112,112,112,112,112,112,112,112,112,112,112,112,112,112,112,112,112,112,112,112,112,112]},"bins": {"c_to_s": [0,0,23,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"s_to_c": [0,0,9,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]},"directions": [0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1],"entropies": [5.610230446,5.622818947,5.669650555,5.651793003,5.651793003,5.604961395,5.645053387,5.630681515,5.633935928,5.622818947,5.633935928,5.669650555,5.651793480,5.645053387,5.669650555,5.683875084,5.669650555,5.701732159,5.633935928,5.666017056,5.633935928,5.666017056,5.645053387,5.677134514,5.637421608,5.672758102,5.602598667,5.623562336,5.651793003,5.683875084,5.669650555,5.701732159]},"ndpi": {"flow_risk": {"17": {"risk":"Malformed Packet","severity":"Low","risk_score": {"total":260,"client":130,"server":130}}},"confidence": {"6":"DPI"},"proto":"ICMP","proto_id":"81","encrypted":0,"breed":"Acceptable","category_id":14,"category":"Network"}}
 01011{"flow_event_id":4,"flow_event_name":"update","thread_id":0,"packet_id":104,"source":"icmp-tunnel.pcap","alias":"nDPId-test","flow_id":1,"flow_state":"finished","flow_src_packets_processed":59,"flow_dst_packets_processed":38,"flow_first_seen":1360227866459330,"flow_src_last_pkt_time":1360227908233821,"flow_dst_last_pkt_time":1360227908233520,"flow_idle_time":140000000,"flow_src_min_l4_payload_len":60,"flow_dst_min_l4_payload_len":0,"flow_src_max_l4_payload_len":852,"flow_dst_max_l4_payload_len":844,"flow_src_tot_l4_payload_len":6481,"flow_dst_tot_l4_payload_len":5677,"midstream":0,"thread_ts_usec":1360227908233821,"l3_proto":"ip4","src_ip":"192.168.154.131","dst_ip":"192.168.154.132","l4_proto":"icmp","flow_datalink":1,"flow_max_packets":3,"ndpi": {"flow_risk": {"17": {"risk":"Malformed Packet","severity":"Low","risk_score": {"total":260,"client":130,"server":130}}},"confidence": {"6":"DPI"},"proto":"ICMP","proto_id":"81","encrypted":0,"breed":"Acceptable","category_id":14,"category":"Network"}}
 01011{"flow_event_id":4,"flow_event_name":"update","thread_id":0,"packet_id":160,"source":"icmp-tunnel.pcap","alias":"nDPId-test","flow_id":1,"flow_state":"finished","flow_src_packets_processed":87,"flow_dst_packets_processed":60,"flow_first_seen":1360227866459330,"flow_src_last_pkt_time":1360228057029534,"flow_dst_last_pkt_time":1360228057029101,"flow_idle_time":140000000,"flow_src_min_l4_payload_len":40,"flow_dst_min_l4_payload_len":0,"flow_src_max_l4_payload_len":852,"flow_dst_max_l4_payload_len":844,"flow_src_tot_l4_payload_len":9088,"flow_dst_tot_l4_payload_len":8105,"midstream":0,"thread_ts_usec":1360228057029534,"l3_proto":"ip4","src_ip":"192.168.154.131","dst_ip":"192.168.154.132","l4_proto":"icmp","flow_datalink":1,"flow_max_packets":3,"ndpi": {"flow_risk": {"17": {"risk":"Malformed Packet","severity":"Low","risk_score": {"total":260,"client":130,"server":130}}},"confidence": {"6":"DPI"},"proto":"ICMP","proto_id":"81","encrypted":0,"breed":"Acceptable","category_id":14,"category":"Network"}}
 01015{"flow_event_id":4,"flow_event_name":"update","thread_id":0,"packet_id":188,"source":"icmp-tunnel.pcap","alias":"nDPId-test","flow_id":1,"flow_state":"finished","flow_src_packets_processed":99,"flow_dst_packets_processed":72,"flow_first_seen":1360227866459330,"flow_src_last_pkt_time":1360228087392773,"flow_dst_last_pkt_time":1360228087392391,"flow_idle_time":140000000,"flow_src_min_l4_payload_len":40,"flow_dst_min_l4_payload_len":0,"flow_src_max_l4_payload_len":1036,"flow_dst_max_l4_payload_len":1036,"flow_src_tot_l4_payload_len":14860,"flow_dst_tot_l4_payload_len":13877,"midstream":0,"thread_ts_usec":1360228087392773,"l3_proto":"ip4","src_ip":"192.168.154.131","dst_ip":"192.168.154.132","l4_proto":"icmp","flow_datalink":1,"flow_max_packets":3,"ndpi": {"flow_risk": {"17": {"risk":"Malformed Packet","severity":"Low","risk_score": {"total":260,"client":130,"server":130}}},"confidence": {"6":"DPI"},"proto":"ICMP","proto_id":"81","encrypted":0,"breed":"Acceptable","category_id":14,"category":"Network"}}
@@ -43,10 +43,10 @@
 ~~ total active/idle flows...: 1/1
 ~~ total timeout flows.......: 0
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-~~ total memory allocated....: 6060668 bytes
-~~ total memory freed........: 6060668 bytes
-~~ total allocations/frees...: 122350/122350
+~~ total memory allocated....: 6060804 bytes
+~~ total memory freed........: 6060804 bytes
+~~ total allocations/frees...: 122351/122351
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 ~~ json string min len.......: 496 chars
-~~ json string max len.......: 1911 chars
-~~ json string avg len.......: 1202 chars
+~~ json string max len.......: 2310 chars
+~~ json string avg len.......: 1402 chars