Added nDPIsrvd-analysed to generate CSV files from analyse events.

 * nDPIsrvd.h: iterate over JSON arrays
 * nDPId: calculate l3 payload packet entropies for analysis

Signed-off-by: Toni Uhlig <matzeton@googlemail.com>

1 files changed, 4 insertions, 4 deletions

diff --git a/test/results/android.pcap.out b/test/results/android.pcap.out
index 1b2fcb2f9..81504d425 100644
--- a/test/results/android.pcap.out
+++ b/test/results/android.pcap.out
@@ -297,7 +297,7 @@
 00550{"packet_event_id":2,"packet_event_name":"packet-flow","thread_id":0,"packet_id":406,"source":"android.pcap","alias":"nDPId-test","flow_id":60,"flow_packet_id":1,"flow_src_last_pkt_time":1582454871881494,"flow_dst_last_pkt_time":1582454871881494,"flow_idle_time":200000000,"pkt_oversize":false,"pkt_caplen":82,"pkt_type":2048,"pkt_l3_offset":14,"pkt_l4_offset":34,"pkt_len":82,"pkt_l4_len":48,"thread_ts_usec":1582454871881494,"pkt":"xiwDYGpkTGr2n\/YnCABFAABErDBAAEARCRfAqAIQwKgCAZtQADUAMNjjuKUBAAABAAAAAAAAB2FuZHJvaWQKZ29vZ2xlYXBpcwNjb20AAAEAAQ=="}
 01013{"flow_event_id":7,"flow_event_name":"detected","thread_id":0,"packet_id":406,"source":"android.pcap","alias":"nDPId-test","flow_id":60,"flow_state":"info","flow_src_packets_processed":1,"flow_dst_packets_processed":0,"flow_first_seen":1582454871881494,"flow_src_last_pkt_time":1582454871881494,"flow_dst_last_pkt_time":1582454871881494,"flow_idle_time":200000000,"flow_src_min_l4_payload_len":40,"flow_dst_min_l4_payload_len":0,"flow_src_max_l4_payload_len":40,"flow_dst_max_l4_payload_len":0,"flow_src_tot_l4_payload_len":40,"flow_dst_tot_l4_payload_len":0,"midstream":0,"thread_ts_usec":1582454871881494,"l3_proto":"ip4","src_ip":"192.168.2.16","dst_ip":"192.168.2.1","src_port":39760,"dst_port":53,"l4_proto":"udp","ndpi": {"confidence": {"6":"DPI"},"proto":"DNS.GoogleServices","proto_id":"5.239","encrypted":0,"breed":"Acceptable","category_id":5,"category":"Web","hostname":"android.googleapis.com","dns": {"num_queries":0,"num_answers":0,"reply_code":0,"query_type":1,"rsp_type":0,"rsp_addr":"0.0.0.0"}}}
 01117{"flow_event_id":7,"flow_event_name":"detected","thread_id":0,"packet_id":408,"source":"android.pcap","alias":"nDPId-test","flow_id":58,"flow_state":"info","flow_src_packets_processed":3,"flow_dst_packets_processed":1,"flow_first_seen":1582454871829800,"flow_src_last_pkt_time":1582454871890562,"flow_dst_last_pkt_time":1582454871867294,"flow_idle_time":3285032704,"flow_src_min_l4_payload_len":0,"flow_dst_min_l4_payload_len":0,"flow_src_max_l4_payload_len":517,"flow_dst_max_l4_payload_len":0,"flow_src_tot_l4_payload_len":517,"flow_dst_tot_l4_payload_len":0,"midstream":0,"thread_ts_usec":1582454871890562,"l3_proto":"ip4","src_ip":"192.168.2.16","dst_ip":"172.217.20.76","src_port":43646,"dst_port":443,"l4_proto":"tcp","ndpi": {"confidence": {"6":"DPI"},"proto":"TLS.DataSaver","proto_id":"91.46","encrypted":1,"breed":"Fun","category_id":5,"category":"Web","hostname":"proxy.googlezip.net","tls": {"version":"TLSv1.2","ja3":"66918128f1b9b03303d77c6f2eefd128","ja3s":"","unsafe_cipher":0,"cipher":"TLS_NULL_WITH_NULL_NULL","alpn":"h2,http\/1.1","tls_supported_versions":"GREASE,TLSv1.3,TLSv1.2,TLSv1.1,TLSv1"}}}
-01716{"flow_event_id":5,"flow_event_name":"analyse","thread_id":0,"packet_id":431,"source":"android.pcap","alias":"nDPId-test","flow_id":42,"flow_state":"finished","flow_src_packets_processed":17,"flow_dst_packets_processed":15,"flow_first_seen":1582454871152402,"flow_src_last_pkt_time":1582454871906464,"flow_dst_last_pkt_time":1582454871901421,"flow_idle_time":3285032704,"flow_src_min_l4_payload_len":0,"flow_dst_min_l4_payload_len":0,"flow_src_max_l4_payload_len":512,"flow_dst_max_l4_payload_len":1418,"flow_src_tot_l4_payload_len":819,"flow_dst_tot_l4_payload_len":10828,"midstream":0,"thread_ts_usec":1582454871906464,"l3_proto":"ip4","src_ip":"192.168.2.16","dst_ip":"216.239.38.120","src_port":32996,"dst_port":443,"l4_proto":"tcp","flow_datalink":1,"flow_max_packets":3,"data_analysis": {"iat": {"min":3,"avg":48486.5,"max":404574,"stddev":104241.1,"var":10866214912.0,"ent":3.0,"data": [13673,15022,32725,47474,16568,3,34518,282,386517,404574,19668,197623,221096,19209,15019,27735,41804,1657,22,36,1002,1575,133,18,9,1204,14,1169,2703,19,10]},"pktlen": {"min":66,"avg":430.5,"max":1484,"stddev":552.7,"var":305506.2,"ent":4.0,"data": [74,74,66,246,66,1484,1202,66,66,159,358,66,578,66,100,66,655,66,1484,1484,1421,1484,66,1484,396,102,66,66,66,66,66,66]},"bins": {"c_to_s": [13,1,1,0,0,1,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"s_to_c": [4,1,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,5,0,0,0]},"directions": [0,1,0,0,1,1,1,0,0,0,1,0,0,1,0,1,1,0,1,1,1,1,0,1,1,1,0,0,0,0,0,0]},"ndpi": {"confidence": {"6":"DPI"},"proto":"TLS.Google","proto_id":"91.126","encrypted":1,"breed":"Acceptable","category_id":5,"category":"Web"}}
+02113{"flow_event_id":5,"flow_event_name":"analyse","thread_id":0,"packet_id":431,"source":"android.pcap","alias":"nDPId-test","flow_id":42,"flow_state":"finished","flow_src_packets_processed":17,"flow_dst_packets_processed":15,"flow_first_seen":1582454871152402,"flow_src_last_pkt_time":1582454871906464,"flow_dst_last_pkt_time":1582454871901421,"flow_idle_time":3285032704,"flow_src_min_l4_payload_len":0,"flow_dst_min_l4_payload_len":0,"flow_src_max_l4_payload_len":512,"flow_dst_max_l4_payload_len":1418,"flow_src_tot_l4_payload_len":819,"flow_dst_tot_l4_payload_len":10828,"midstream":0,"thread_ts_usec":1582454871906464,"l3_proto":"ip4","src_ip":"192.168.2.16","dst_ip":"216.239.38.120","src_port":32996,"dst_port":443,"l4_proto":"tcp","flow_datalink":1,"flow_max_packets":3,"data_analysis": {"iat": {"min":3,"avg":48486.5,"max":404574,"stddev":104241.1,"var":10866214912.0,"ent":3.0,"data": [13673,15022,32725,47474,16568,3,34518,282,386517,404574,19668,197623,221096,19209,15019,27735,41804,1657,22,36,1002,1575,133,18,9,1204,14,1169,2703,19,10]},"pktlen": {"min":52,"avg":416.5,"max":1470,"stddev":552.7,"var":305506.2,"ent":3.9,"data": [60,60,52,232,52,1470,1188,52,52,145,344,52,564,52,86,52,641,52,1470,1470,1407,1470,52,1470,382,88,52,52,52,52,52,52]},"bins": {"c_to_s": [13,1,1,0,0,1,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"s_to_c": [4,1,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,5,0,0,0]},"directions": [0,1,0,0,1,1,1,0,0,0,1,0,0,1,0,1,1,0,1,1,1,1,0,1,1,1,0,0,0,0,0,0],"entropies": [4.671797276,5.277319908,5.092563152,5.518131256,5.077241421,7.236341000,7.433474064,5.131024837,5.131024837,6.086913109,7.119209766,4.962661266,7.515064716,4.947339535,5.439514160,5.038779736,7.633175850,5.015639782,7.866302967,7.846067905,7.867026806,7.835390091,5.092563152,7.847195148,7.413039684,5.580356598,5.054101467,5.092563152,5.054101467,5.092563152,5.015639782,4.977178097]},"ndpi": {"confidence": {"6":"DPI"},"proto":"TLS.Google","proto_id":"91.126","encrypted":1,"breed":"Acceptable","category_id":5,"category":"Web"}}
 01163{"flow_event_id":8,"flow_event_name":"detection-update","thread_id":0,"packet_id":434,"source":"android.pcap","alias":"nDPId-test","flow_id":59,"flow_state":"info","flow_src_packets_processed":3,"flow_dst_packets_processed":3,"flow_first_seen":1582454871839297,"flow_src_last_pkt_time":1582454871880409,"flow_dst_last_pkt_time":1582454871911317,"flow_idle_time":3285032704,"flow_src_min_l4_payload_len":0,"flow_dst_min_l4_payload_len":0,"flow_src_max_l4_payload_len":517,"flow_dst_max_l4_payload_len":1418,"flow_src_tot_l4_payload_len":517,"flow_dst_tot_l4_payload_len":1418,"midstream":0,"thread_ts_usec":1582454871911317,"l3_proto":"ip4","src_ip":"192.168.2.16","dst_ip":"216.239.38.120","src_port":33014,"dst_port":443,"l4_proto":"tcp","ndpi": {"confidence": {"6":"DPI"},"proto":"TLS.Google","proto_id":"91.126","encrypted":1,"breed":"Acceptable","category_id":5,"category":"Web","hostname":"www.google.com","tls": {"version":"TLSv1.3","ja3":"66918128f1b9b03303d77c6f2eefd128","ja3s":"eb1d94daa7e0344597e756a1fb6e7054","unsafe_cipher":0,"cipher":"TLS_AES_128_GCM_SHA256","alpn":"h2,http\/1.1","tls_supported_versions":"GREASE,TLSv1.3,TLSv1.2,TLSv1.1,TLSv1"}}}
 01166{"flow_event_id":8,"flow_event_name":"detection-update","thread_id":0,"packet_id":437,"source":"android.pcap","alias":"nDPId-test","flow_id":55,"flow_state":"info","flow_src_packets_processed":3,"flow_dst_packets_processed":3,"flow_first_seen":1582454871814833,"flow_src_last_pkt_time":1582454871879681,"flow_dst_last_pkt_time":1582454871913572,"flow_idle_time":3285032704,"flow_src_min_l4_payload_len":0,"flow_dst_min_l4_payload_len":0,"flow_src_max_l4_payload_len":594,"flow_dst_max_l4_payload_len":212,"flow_src_tot_l4_payload_len":594,"flow_dst_tot_l4_payload_len":212,"midstream":0,"thread_ts_usec":1582454871913572,"l3_proto":"ip4","src_ip":"192.168.2.16","dst_ip":"172.217.21.202","src_port":51944,"dst_port":443,"l4_proto":"tcp","ndpi": {"confidence": {"6":"DPI"},"proto":"TLS.DataSaver","proto_id":"91.46","encrypted":1,"breed":"Fun","category_id":5,"category":"Web","hostname":"datasaver.googleapis.com","tls": {"version":"TLSv1.3","ja3":"554719594ba90b02ae410c297c6e50ad","ja3s":"2b0648ab686ee45e0e7c35fcfb0eea7e","unsafe_cipher":0,"cipher":"TLS_AES_128_GCM_SHA256","alpn":"h2,http\/1.1","tls_supported_versions":"GREASE,TLSv1.3,TLSv1.2,TLSv1.1,TLSv1"}}}
 00570{"packet_event_id":2,"packet_event_name":"packet-flow","thread_id":0,"packet_id":441,"source":"android.pcap","alias":"nDPId-test","flow_id":60,"flow_packet_id":2,"flow_src_last_pkt_time":1582454871881494,"flow_dst_last_pkt_time":1582454871920611,"flow_idle_time":200000000,"pkt_oversize":false,"pkt_caplen":98,"pkt_type":2048,"pkt_l3_offset":14,"pkt_l4_offset":34,"pkt_len":98,"pkt_l4_len":64,"thread_ts_usec":1582454871920611,"pkt":"TGr2n\/YnxiwDYGpkCABFAABUFXQAAEAR38PAqAIBwKgCEAA1m1AAQNQ0uKWBgAABAAEAAAAAB2FuZHJvaWQKZ29vZ2xlYXBpcwNjb20AAAEAAcAMAAEAAQAAARcABKzZFgo="}
@@ -392,9 +392,9 @@
 ~~ total active/idle flows...: 63/63
 ~~ total timeout flows.......: 0
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-~~ total memory allocated....: 6392351 bytes
-~~ total memory freed........: 6392351 bytes
-~~ total allocations/frees...: 122858/122858
+~~ total memory allocated....: 6400919 bytes
+~~ total memory freed........: 6400919 bytes
+~~ total allocations/frees...: 122921/122921
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 ~~ json string min len.......: 492 chars
 ~~ json string max len.......: 2570 chars