Added nDPIsrvd-analysed to generate CSV files from analyse events.

 * nDPIsrvd.h: iterate over JSON arrays
 * nDPId: calculate l3 payload packet entropies for analysis

Signed-off-by: Toni Uhlig <matzeton@googlemail.com>

1 files changed, 5 insertions, 5 deletions

diff --git a/test/results/vxlan.pcap.out b/test/results/vxlan.pcap.out
index 0dc90fb2b..9dee685de 100644
--- a/test/results/vxlan.pcap.out
+++ b/test/results/vxlan.pcap.out
@@ -41,8 +41,8 @@
 00860{"flow_event_id":7,"flow_event_name":"detected","thread_id":0,"packet_id":54,"source":"vxlan.pcap","alias":"nDPId-test","flow_id":9,"flow_state":"info","flow_src_packets_processed":1,"flow_dst_packets_processed":0,"flow_first_seen":1639650443097770,"flow_src_last_pkt_time":1639650443097770,"flow_dst_last_pkt_time":1639650443097770,"flow_idle_time":200000000,"flow_src_min_l4_payload_len":62,"flow_dst_min_l4_payload_len":0,"flow_src_max_l4_payload_len":62,"flow_dst_max_l4_payload_len":0,"flow_src_tot_l4_payload_len":62,"flow_dst_tot_l4_payload_len":0,"midstream":0,"thread_ts_usec":1639650443097770,"l3_proto":"ip4","src_ip":"192.168.22.4","dst_ip":"192.168.22.5","src_port":60230,"dst_port":4789,"l4_proto":"udp","ndpi": {"confidence": {"6":"DPI"},"proto":"VXLAN","proto_id":"64","encrypted":0,"breed":"Acceptable","category_id":14,"category":"Network"}}
 00580{"packet_event_id":2,"packet_event_name":"packet-flow","thread_id":0,"packet_id":55,"source":"vxlan.pcap","alias":"nDPId-test","flow_id":9,"flow_packet_id":2,"flow_src_last_pkt_time":1639650443097913,"flow_dst_last_pkt_time":1639650443097770,"flow_idle_time":200000000,"pkt_oversize":false,"pkt_caplen":108,"pkt_type":2048,"pkt_l3_offset":18,"pkt_l4_offset":38,"pkt_len":108,"pkt_l4_len":70,"thread_ts_usec":1639650443097913,"pkt":"AAy9Bjp0AAy9Bjp1gQAABQgARQAAWgOJAABAEcmwwKgWBMCoFgXrRhK1AEbaoAgAAAAABFcAZnpQqv+aHuppKm\/PCABFCAAoAABAAEAGnqYKChQEnfDgI7CqAbtGa9YTAAAAAFAEAABE2gAA"}
 00580{"packet_event_id":2,"packet_event_name":"packet-flow","thread_id":0,"packet_id":56,"source":"vxlan.pcap","alias":"nDPId-test","flow_id":9,"flow_packet_id":3,"flow_src_last_pkt_time":1639650443097920,"flow_dst_last_pkt_time":1639650443097770,"flow_idle_time":200000000,"pkt_oversize":false,"pkt_caplen":108,"pkt_type":2048,"pkt_l3_offset":18,"pkt_l4_offset":38,"pkt_len":108,"pkt_l4_len":70,"thread_ts_usec":1639650443097920,"pkt":"AAy9Bjp0AAy9Bjp1gQAABQgARQAAWgOKAABAEcmvwKgWBMCoFgXrRhK1AEbaoAgAAAAABFcAZnpQqv+aHuppKm\/PCABFCAAoAABAAEAGnqYKChQEnfDgI7CqAbtGa9YUAAAAAFAEAABE2QAA"}
-01722{"flow_event_id":5,"flow_event_name":"analyse","thread_id":0,"packet_id":89,"source":"vxlan.pcap","alias":"nDPId-test","flow_id":8,"flow_state":"finished","flow_src_packets_processed":32,"flow_dst_packets_processed":0,"flow_first_seen":1639650442941597,"flow_src_last_pkt_time":1639650443255719,"flow_dst_last_pkt_time":1639650442941597,"flow_idle_time":200000000,"flow_src_min_l4_payload_len":74,"flow_dst_min_l4_payload_len":0,"flow_src_max_l4_payload_len":1454,"flow_dst_max_l4_payload_len":0,"flow_src_tot_l4_payload_len":35959,"flow_dst_tot_l4_payload_len":0,"midstream":0,"thread_ts_usec":1639650443255719,"l3_proto":"ip4","src_ip":"192.168.22.5","dst_ip":"192.168.22.4","src_port":36286,"dst_port":4789,"l4_proto":"udp","flow_datalink":1,"flow_max_packets":3,"data_analysis": {"iat": {"min":10,"avg":10133.0,"max":140558,"stddev":31047.2,"var":963930240.0,"ent":2.2,"data": [10532,1402,105,10,11439,530,9521,113264,10571,140558,101,64,3057,190,558,175,1284,181,1316,3621,187,402,189,2282,184,313,186,833,189,694,184]},"pktlen": {"min":120,"avg":1169.7,"max":1500,"stddev":546.6,"var":298767.6,"ent":4.8,"data": [128,120,1500,1500,588,120,289,120,572,120,1500,1500,874,1500,1500,1500,1500,1500,1500,1500,1500,1500,1500,1500,1500,1500,1500,1500,1500,1500,1500,1500]},"bins": {"c_to_s": [0,0,5,0,0,0,0,1,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,23,0,0],"s_to_c": [0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]},"directions": [0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]},"ndpi": {"confidence": {"6":"DPI"},"proto":"VXLAN","proto_id":"64","encrypted":0,"breed":"Acceptable","category_id":14,"category":"Network"}}
-01695{"flow_event_id":5,"flow_event_name":"analyse","thread_id":0,"packet_id":122,"source":"vxlan.pcap","alias":"nDPId-test","flow_id":7,"flow_state":"finished","flow_src_packets_processed":32,"flow_dst_packets_processed":0,"flow_first_seen":1639650442931548,"flow_src_last_pkt_time":1639650443264733,"flow_dst_last_pkt_time":1639650442931548,"flow_idle_time":200000000,"flow_src_min_l4_payload_len":74,"flow_dst_min_l4_payload_len":0,"flow_src_max_l4_payload_len":392,"flow_dst_max_l4_payload_len":0,"flow_src_tot_l4_payload_len":3106,"flow_dst_tot_l4_payload_len":0,"midstream":0,"thread_ts_usec":1639650443264733,"l3_proto":"ip4","src_ip":"192.168.22.4","dst_ip":"192.168.22.5","src_port":40646,"dst_port":4789,"l4_proto":"udp","flow_datalink":1,"flow_max_packets":3,"data_analysis": {"iat": {"min":4,"avg":10747.9,"max":150839,"stddev":30032.6,"var":901957440.0,"ent":2.5,"data": [10329,305,11530,200,4,1301,10031,41817,81536,403,150839,3109,802,1504,1403,3811,602,2508,504,1003,903,802,707,803,710,2107,301,402,2307,401,201]},"pktlen": {"min":120,"avg":143.1,"max":438,"stddev":68.2,"var":4655.6,"ent":4.9,"data": [128,120,438,120,120,120,184,285,120,120,303,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120]},"bins": {"c_to_s": [0,0,28,0,1,0,0,1,1,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"s_to_c": [0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]},"directions": [0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]},"ndpi": {"confidence": {"6":"DPI"},"proto":"VXLAN","proto_id":"64","encrypted":0,"breed":"Acceptable","category_id":14,"category":"Network"}}
+02121{"flow_event_id":5,"flow_event_name":"analyse","thread_id":0,"packet_id":89,"source":"vxlan.pcap","alias":"nDPId-test","flow_id":8,"flow_state":"finished","flow_src_packets_processed":32,"flow_dst_packets_processed":0,"flow_first_seen":1639650442941597,"flow_src_last_pkt_time":1639650443255719,"flow_dst_last_pkt_time":1639650442941597,"flow_idle_time":200000000,"flow_src_min_l4_payload_len":74,"flow_dst_min_l4_payload_len":0,"flow_src_max_l4_payload_len":1454,"flow_dst_max_l4_payload_len":0,"flow_src_tot_l4_payload_len":35959,"flow_dst_tot_l4_payload_len":0,"midstream":0,"thread_ts_usec":1639650443255719,"l3_proto":"ip4","src_ip":"192.168.22.5","dst_ip":"192.168.22.4","src_port":36286,"dst_port":4789,"l4_proto":"udp","flow_datalink":1,"flow_max_packets":3,"data_analysis": {"iat": {"min":10,"avg":10133.0,"max":140558,"stddev":31047.2,"var":963930240.0,"ent":2.2,"data": [10532,1402,105,10,11439,530,9521,113264,10571,140558,101,64,3057,190,558,175,1284,181,1316,3621,187,402,189,2282,184,313,186,833,189,694,184]},"pktlen": {"min":102,"avg":1151.7,"max":1482,"stddev":546.6,"var":298767.6,"ent":4.8,"data": [110,102,1482,1482,570,102,271,102,554,102,1482,1482,856,1482,1482,1482,1482,1482,1482,1482,1482,1482,1482,1482,1482,1482,1482,1482,1482,1482,1482,1482]},"bins": {"c_to_s": [0,0,5,0,0,0,0,1,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,23,0,0],"s_to_c": [0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]},"directions": [0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"entropies": [5.583852291,5.651705265,7.826985836,7.861832619,7.623077869,5.619890690,7.052967072,5.635924816,7.564305782,5.565874100,7.866837978,7.859116077,7.762131214,7.859333515,7.877618790,7.863654613,7.851696491,7.874659538,7.855105877,7.845957756,7.883800030,7.862126827,7.878228188,7.846958637,7.850887299,7.866386890,7.866912842,7.871983051,7.852091789,7.857552052,7.852843761,7.854843616]},"ndpi": {"confidence": {"6":"DPI"},"proto":"VXLAN","proto_id":"64","encrypted":0,"breed":"Acceptable","category_id":14,"category":"Network"}}
+02094{"flow_event_id":5,"flow_event_name":"analyse","thread_id":0,"packet_id":122,"source":"vxlan.pcap","alias":"nDPId-test","flow_id":7,"flow_state":"finished","flow_src_packets_processed":32,"flow_dst_packets_processed":0,"flow_first_seen":1639650442931548,"flow_src_last_pkt_time":1639650443264733,"flow_dst_last_pkt_time":1639650442931548,"flow_idle_time":200000000,"flow_src_min_l4_payload_len":74,"flow_dst_min_l4_payload_len":0,"flow_src_max_l4_payload_len":392,"flow_dst_max_l4_payload_len":0,"flow_src_tot_l4_payload_len":3106,"flow_dst_tot_l4_payload_len":0,"midstream":0,"thread_ts_usec":1639650443264733,"l3_proto":"ip4","src_ip":"192.168.22.4","dst_ip":"192.168.22.5","src_port":40646,"dst_port":4789,"l4_proto":"udp","flow_datalink":1,"flow_max_packets":3,"data_analysis": {"iat": {"min":4,"avg":10747.9,"max":150839,"stddev":30032.6,"var":901957440.0,"ent":2.5,"data": [10329,305,11530,200,4,1301,10031,41817,81536,403,150839,3109,802,1504,1403,3811,602,2508,504,1003,903,802,707,803,710,2107,301,402,2307,401,201]},"pktlen": {"min":102,"avg":125.1,"max":420,"stddev":68.2,"var":4655.6,"ent":4.8,"data": [110,102,420,102,102,102,166,267,102,102,285,102,102,102,102,102,102,102,102,102,102,102,102,102,102,102,102,102,102,102,102,102]},"bins": {"c_to_s": [0,0,28,0,1,0,0,1,1,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"s_to_c": [0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]},"directions": [0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"entropies": [5.313875198,5.603603840,6.154091835,5.623211861,5.630611897,5.623211384,6.288531303,6.880884647,5.615810394,5.596202850,7.036987305,5.564387798,5.603603840,5.596202850,5.623211384,5.564388275,5.583995819,5.556987286,5.591396332,5.603603840,5.576594353,5.623211384,5.544780254,5.603603840,5.603603840,5.623211384,5.642818928,5.588801384,5.603603363,5.635418415,5.635418415,5.655025959]},"ndpi": {"confidence": {"6":"DPI"},"proto":"VXLAN","proto_id":"64","encrypted":0,"breed":"Acceptable","category_id":14,"category":"Network"}}
 00901{"flow_event_id":3,"flow_event_name":"idle","thread_id":0,"packet_id":127,"source":"vxlan.pcap","alias":"nDPId-test","flow_id":5,"flow_state":"finished","flow_src_packets_processed":2,"flow_dst_packets_processed":0,"flow_first_seen":1639650442864784,"flow_src_last_pkt_time":1639650442864881,"flow_dst_last_pkt_time":1639650442864784,"flow_idle_time":200000000,"flow_src_min_l4_payload_len":84,"flow_dst_min_l4_payload_len":0,"flow_src_max_l4_payload_len":84,"flow_dst_max_l4_payload_len":0,"flow_src_tot_l4_payload_len":168,"flow_dst_tot_l4_payload_len":0,"midstream":0,"thread_ts_usec":1639650443276366,"l3_proto":"ip4","src_ip":"192.168.22.4","dst_ip":"192.168.22.5","src_port":60351,"dst_port":4789,"l4_proto":"udp","flow_datalink":1,"flow_max_packets":3,"ndpi": {"confidence": {"6":"DPI"},"proto":"VXLAN","proto_id":"64","encrypted":0,"breed":"Acceptable","category_id":14,"category":"Network"}}
 00903{"flow_event_id":3,"flow_event_name":"idle","thread_id":0,"packet_id":127,"source":"vxlan.pcap","alias":"nDPId-test","flow_id":6,"flow_state":"finished","flow_src_packets_processed":2,"flow_dst_packets_processed":0,"flow_first_seen":1639650442902284,"flow_src_last_pkt_time":1639650442930989,"flow_dst_last_pkt_time":1639650442902284,"flow_idle_time":200000000,"flow_src_min_l4_payload_len":129,"flow_dst_min_l4_payload_len":0,"flow_src_max_l4_payload_len":141,"flow_dst_max_l4_payload_len":0,"flow_src_tot_l4_payload_len":270,"flow_dst_tot_l4_payload_len":0,"midstream":0,"thread_ts_usec":1639650443276366,"l3_proto":"ip4","src_ip":"192.168.22.5","dst_ip":"192.168.22.4","src_port":50251,"dst_port":4789,"l4_proto":"udp","flow_datalink":1,"flow_max_packets":3,"ndpi": {"confidence": {"6":"DPI"},"proto":"VXLAN","proto_id":"64","encrypted":0,"breed":"Acceptable","category_id":14,"category":"Network"}}
 00906{"flow_event_id":3,"flow_event_name":"idle","thread_id":0,"packet_id":127,"source":"vxlan.pcap","alias":"nDPId-test","flow_id":8,"flow_state":"finished","flow_src_packets_processed":56,"flow_dst_packets_processed":0,"flow_first_seen":1639650442941597,"flow_src_last_pkt_time":1639650443276182,"flow_dst_last_pkt_time":1639650442941597,"flow_idle_time":200000000,"flow_src_min_l4_payload_len":74,"flow_dst_min_l4_payload_len":0,"flow_src_max_l4_payload_len":1454,"flow_dst_max_l4_payload_len":0,"flow_src_tot_l4_payload_len":68647,"flow_dst_tot_l4_payload_len":0,"midstream":0,"thread_ts_usec":1639650443276366,"l3_proto":"ip4","src_ip":"192.168.22.5","dst_ip":"192.168.22.4","src_port":36286,"dst_port":4789,"l4_proto":"udp","flow_datalink":1,"flow_max_packets":3,"ndpi": {"confidence": {"6":"DPI"},"proto":"VXLAN","proto_id":"64","encrypted":0,"breed":"Acceptable","category_id":14,"category":"Network"}}
@@ -61,9 +61,9 @@
 ~~ total active/idle flows...: 9/9
 ~~ total timeout flows.......: 0
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-~~ total memory allocated....: 6052316 bytes
-~~ total memory freed........: 6052316 bytes
-~~ total allocations/frees...: 121694/121694
+~~ total memory allocated....: 6053540 bytes
+~~ total memory freed........: 6053540 bytes
+~~ total allocations/frees...: 121703/121703
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 ~~ json string min len.......: 490 chars
 ~~ json string max len.......: 2471 chars