Added nDPIsrvd-analysed to generate CSV files from analyse events.

 * nDPIsrvd.h: iterate over JSON arrays
 * nDPId: calculate l3 payload packet entropies for analysis

Signed-off-by: Toni Uhlig <matzeton@googlemail.com>

1 files changed, 6 insertions, 6 deletions

diff --git a/test/results/rx.pcap.out b/test/results/rx.pcap.out
index ce7f4115a..e00fa6a34 100644
--- a/test/results/rx.pcap.out
+++ b/test/results/rx.pcap.out
@@ -25,7 +25,7 @@
 00576{"packet_event_id":2,"packet_event_name":"packet-flow","thread_id":0,"packet_id":28,"source":"rx.pcap","alias":"nDPId-test","flow_id":5,"flow_packet_id":2,"flow_src_last_pkt_time":1460647299986990,"flow_dst_last_pkt_time":1460647300017623,"flow_idle_time":200000000,"pkt_oversize":false,"pkt_caplen":108,"pkt_type":2048,"pkt_l3_offset":14,"pkt_l4_offset":34,"pkt_len":108,"pkt_l4_len":74,"thread_ts_usec":1460647300017623,"pkt":"AAjK968mPIqwbTfwCABFAABeUWIAADoRQO7Ap858g3LbqBtYG1kASjJ01w+zMFwiT7QAAAABAAAAAAAAAAECIgAAXV0AAQAAAAAAAAABAAAAAQAAAAAGAQEAAAAAAAWkAAAFpAAAABAAAAAB"}
 00858{"flow_event_id":7,"flow_event_name":"detected","thread_id":0,"packet_id":28,"source":"rx.pcap","alias":"nDPId-test","flow_id":5,"flow_state":"info","flow_src_packets_processed":1,"flow_dst_packets_processed":1,"flow_first_seen":1460647299986990,"flow_src_last_pkt_time":1460647299986990,"flow_dst_last_pkt_time":1460647300017623,"flow_idle_time":200000000,"flow_src_min_l4_payload_len":32,"flow_dst_min_l4_payload_len":0,"flow_src_max_l4_payload_len":32,"flow_dst_max_l4_payload_len":66,"flow_src_tot_l4_payload_len":32,"flow_dst_tot_l4_payload_len":66,"midstream":0,"thread_ts_usec":1460647300017623,"l3_proto":"ip4","src_ip":"131.114.219.168","dst_ip":"192.167.206.124","src_port":7001,"dst_port":7000,"l4_proto":"udp","ndpi": {"confidence": {"6":"DPI"},"proto":"RX","proto_id":"223","encrypted":0,"breed":"Acceptable","category_id":16,"category":"RPC"}}
 00576{"packet_event_id":2,"packet_event_name":"packet-flow","thread_id":0,"packet_id":29,"source":"rx.pcap","alias":"nDPId-test","flow_id":5,"flow_packet_id":3,"flow_src_last_pkt_time":1460647300017672,"flow_dst_last_pkt_time":1460647300017623,"flow_idle_time":200000000,"pkt_oversize":false,"pkt_caplen":107,"pkt_type":2048,"pkt_l3_offset":14,"pkt_l4_offset":34,"pkt_len":107,"pkt_l4_len":73,"thread_ts_usec":1460647300017672,"pkt":"PIqwbTfwAAjK968mCABFAABd9xIAAEARlT6DctuowKfOfBtZG1gASacR1w+zMFwiT7QAAAABAAAAAAAAAAICIQAAAAAAAQAAAAAAAAABAAAAAAAAAAEHAAAAAAAAFjwAAAWkAAAAEAAAAAQ="}
-01707{"flow_event_id":5,"flow_event_name":"analyse","thread_id":0,"packet_id":61,"source":"rx.pcap","alias":"nDPId-test","flow_id":4,"flow_state":"finished","flow_src_packets_processed":17,"flow_dst_packets_processed":15,"flow_first_seen":1460647299704750,"flow_src_last_pkt_time":1460647300147650,"flow_dst_last_pkt_time":1460647300150407,"flow_idle_time":200000000,"flow_src_min_l4_payload_len":28,"flow_dst_min_l4_payload_len":0,"flow_src_max_l4_payload_len":468,"flow_dst_max_l4_payload_len":740,"flow_src_tot_l4_payload_len":2528,"flow_dst_tot_l4_payload_len":1781,"midstream":0,"thread_ts_usec":1460647300150407,"l3_proto":"ip4","src_ip":"131.114.219.168","dst_ip":"192.167.206.241","src_port":7001,"dst_port":7000,"l4_proto":"udp","flow_datalink":1,"flow_max_packets":3,"data_analysis": {"iat": {"min":52,"avg":28663.1,"max":105287,"stddev":33586.2,"var":1128029952.0,"ent":4.0,"data": [77545,77601,57048,57152,38155,1292,39484,65722,277,65926,103176,105287,2087,8975,9068,2966,1842,4798,61436,65225,3784,52,6802,6683,61,3692,3703,4895,8042,2994,2787]},"pktlen": {"min":70,"avg":176.7,"max":782,"stddev":165.9,"var":27529.2,"ent":4.5,"data": [74,108,107,74,510,107,118,70,107,78,107,94,86,435,74,510,107,198,107,174,782,107,94,198,107,110,214,107,94,86,435,74]},"bins": {"c_to_s": [1,4,7,0,1,0,0,0,0,0,0,0,2,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"s_to_c": [0,6,5,0,2,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]},"directions": [0,1,0,1,0,1,1,0,1,1,0,0,1,0,1,0,1,1,0,0,1,0,0,1,0,0,1,0,0,1,0,1]},"ndpi": {"confidence": {"6":"DPI"},"proto":"RX","proto_id":"223","encrypted":0,"breed":"Acceptable","category_id":16,"category":"RPC"}}
+02095{"flow_event_id":5,"flow_event_name":"analyse","thread_id":0,"packet_id":61,"source":"rx.pcap","alias":"nDPId-test","flow_id":4,"flow_state":"finished","flow_src_packets_processed":17,"flow_dst_packets_processed":15,"flow_first_seen":1460647299704750,"flow_src_last_pkt_time":1460647300147650,"flow_dst_last_pkt_time":1460647300150407,"flow_idle_time":200000000,"flow_src_min_l4_payload_len":28,"flow_dst_min_l4_payload_len":0,"flow_src_max_l4_payload_len":468,"flow_dst_max_l4_payload_len":740,"flow_src_tot_l4_payload_len":2528,"flow_dst_tot_l4_payload_len":1781,"midstream":0,"thread_ts_usec":1460647300150407,"l3_proto":"ip4","src_ip":"131.114.219.168","dst_ip":"192.167.206.241","src_port":7001,"dst_port":7000,"l4_proto":"udp","flow_datalink":1,"flow_max_packets":3,"data_analysis": {"iat": {"min":52,"avg":28663.1,"max":105287,"stddev":33586.2,"var":1128029952.0,"ent":4.0,"data": [77545,77601,57048,57152,38155,1292,39484,65722,277,65926,103176,105287,2087,8975,9068,2966,1842,4798,61436,65225,3784,52,6802,6683,61,3692,3703,4895,8042,2994,2787]},"pktlen": {"min":56,"avg":162.7,"max":768,"stddev":165.9,"var":27529.2,"ent":4.5,"data": [60,94,93,60,496,93,104,56,93,64,93,80,72,421,60,496,93,184,93,160,768,93,80,184,93,96,200,93,80,72,421,60]},"bins": {"c_to_s": [1,4,7,0,1,0,0,0,0,0,0,0,2,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"s_to_c": [0,6,5,0,2,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]},"directions": [0,1,0,1,0,1,1,0,1,1,0,0,1,0,1,0,1,1,0,0,1,0,0,1,0,0,1,0,0,1,0,1],"entropies": [4.077751637,3.436867237,3.527563810,4.011084557,4.341524601,3.511269569,3.900409222,4.138328075,3.532774925,3.942092896,3.562457323,5.267373562,3.808812857,7.113327503,4.069581032,4.336879253,3.522217751,6.534686565,3.586733341,6.423340321,7.662765026,3.559956789,5.237494469,6.512056828,3.573345423,5.590069771,6.656118393,3.594850540,5.217373848,3.930941820,7.131436825,4.136247635]},"ndpi": {"confidence": {"6":"DPI"},"proto":"RX","proto_id":"223","encrypted":0,"breed":"Acceptable","category_id":16,"category":"RPC"}}
 00901{"flow_event_id":3,"flow_event_name":"idle","thread_id":0,"packet_id":132,"source":"rx.pcap","alias":"nDPId-test","flow_id":1,"flow_state":"finished","flow_src_packets_processed":2,"flow_dst_packets_processed":1,"flow_first_seen":1460647264018403,"flow_src_last_pkt_time":1460647264026325,"flow_dst_last_pkt_time":1460647264026287,"flow_idle_time":200000000,"flow_src_min_l4_payload_len":65,"flow_dst_min_l4_payload_len":0,"flow_src_max_l4_payload_len":292,"flow_dst_max_l4_payload_len":36,"flow_src_tot_l4_payload_len":357,"flow_dst_tot_l4_payload_len":36,"midstream":0,"thread_ts_usec":1460647320158051,"l3_proto":"ip4","src_ip":"131.114.219.168","dst_ip":"192.167.206.124","src_port":41559,"dst_port":7002,"l4_proto":"udp","flow_datalink":1,"flow_max_packets":3,"ndpi": {"confidence": {"6":"DPI"},"proto":"RX","proto_id":"223","encrypted":0,"breed":"Acceptable","category_id":16,"category":"RPC"}}
 00905{"flow_event_id":3,"flow_event_name":"idle","thread_id":0,"packet_id":132,"source":"rx.pcap","alias":"nDPId-test","flow_id":5,"flow_state":"finished","flow_src_packets_processed":10,"flow_dst_packets_processed":10,"flow_first_seen":1460647299986990,"flow_src_last_pkt_time":1460647320158051,"flow_dst_last_pkt_time":1460647300312692,"flow_idle_time":200000000,"flow_src_min_l4_payload_len":28,"flow_dst_min_l4_payload_len":0,"flow_src_max_l4_payload_len":468,"flow_dst_max_l4_payload_len":156,"flow_src_tot_l4_payload_len":1665,"flow_dst_tot_l4_payload_len":637,"midstream":0,"thread_ts_usec":1460647320158051,"l3_proto":"ip4","src_ip":"131.114.219.168","dst_ip":"192.167.206.124","src_port":7001,"dst_port":7000,"l4_proto":"udp","flow_datalink":1,"flow_max_packets":3,"ndpi": {"confidence": {"6":"DPI"},"proto":"RX","proto_id":"223","encrypted":0,"breed":"Acceptable","category_id":16,"category":"RPC"}}
 00906{"flow_event_id":3,"flow_event_name":"idle","thread_id":0,"packet_id":132,"source":"rx.pcap","alias":"nDPId-test","flow_id":4,"flow_state":"finished","flow_src_packets_processed":48,"flow_dst_packets_processed":31,"flow_first_seen":1460647299704750,"flow_src_last_pkt_time":1460647320158014,"flow_dst_last_pkt_time":1460647300329629,"flow_idle_time":200000000,"flow_src_min_l4_payload_len":28,"flow_dst_min_l4_payload_len":0,"flow_src_max_l4_payload_len":468,"flow_dst_max_l4_payload_len":740,"flow_src_tot_l4_payload_len":4792,"flow_dst_tot_l4_payload_len":4266,"midstream":0,"thread_ts_usec":1460647320158051,"l3_proto":"ip4","src_ip":"131.114.219.168","dst_ip":"192.167.206.241","src_port":7001,"dst_port":7000,"l4_proto":"udp","flow_datalink":1,"flow_max_packets":3,"ndpi": {"confidence": {"6":"DPI"},"proto":"RX","proto_id":"223","encrypted":0,"breed":"Acceptable","category_id":16,"category":"RPC"}}
@@ -40,10 +40,10 @@
 ~~ total active/idle flows...: 5/5
 ~~ total timeout flows.......: 0
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-~~ total memory allocated....: 6045965 bytes
-~~ total memory freed........: 6045965 bytes
-~~ total allocations/frees...: 121659/121659
+~~ total memory allocated....: 6046645 bytes
+~~ total memory freed........: 6046645 bytes
+~~ total allocations/frees...: 121664/121664
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 ~~ json string min len.......: 487 chars
-~~ json string max len.......: 1958 chars
-~~ json string avg len.......: 1221 chars
+~~ json string max len.......: 2100 chars
+~~ json string avg len.......: 1292 chars