Added nDPIsrvd-analysed to generate CSV files from analyse events.

 * nDPIsrvd.h: iterate over JSON arrays
 * nDPId: calculate l3 payload packet entropies for analysis

Signed-off-by: Toni Uhlig <matzeton@googlemail.com>

1 files changed, 6 insertions, 6 deletions

diff --git a/test/results/443-git.pcap.out b/test/results/443-git.pcap.out
index d228ad538..1048a7ea4 100644
--- a/test/results/443-git.pcap.out
+++ b/test/results/443-git.pcap.out
@@ -7,7 +7,7 @@
 01053{"flow_event_id":7,"flow_event_name":"detected","thread_id":0,"packet_id":4,"source":"443-git.pcap","alias":"nDPId-test","flow_id":1,"flow_state":"info","flow_src_packets_processed":3,"flow_dst_packets_processed":1,"flow_first_seen":1581113657633853,"flow_src_last_pkt_time":1581113657751016,"flow_dst_last_pkt_time":1581113657744320,"flow_idle_time":3285032704,"flow_src_min_l4_payload_len":0,"flow_dst_min_l4_payload_len":0,"flow_src_max_l4_payload_len":517,"flow_dst_max_l4_payload_len":0,"flow_src_tot_l4_payload_len":517,"flow_dst_tot_l4_payload_len":0,"midstream":0,"thread_ts_usec":1581113657751016,"l3_proto":"ip4","src_ip":"192.168.1.13","dst_ip":"140.82.114.4","src_port":55744,"dst_port":443,"l4_proto":"tcp","ndpi": {"confidence": {"6":"DPI"},"proto":"TLS.Github","proto_id":"91.203","encrypted":1,"breed":"Acceptable","category_id":15,"category":"Collaborative","hostname":"github.com","tls": {"version":"TLSv1.2","ja3":"2a26b1a62e40d25d4de3babc9d532f30","ja3s":"","unsafe_cipher":0,"cipher":"TLS_NULL_WITH_NULL_NULL","alpn":"http\/1.1"}}}
 01113{"flow_event_id":8,"flow_event_name":"detection-update","thread_id":0,"packet_id":5,"source":"443-git.pcap","alias":"nDPId-test","flow_id":1,"flow_state":"info","flow_src_packets_processed":3,"flow_dst_packets_processed":2,"flow_first_seen":1581113657633853,"flow_src_last_pkt_time":1581113657751016,"flow_dst_last_pkt_time":1581113657863699,"flow_idle_time":3285032704,"flow_src_min_l4_payload_len":0,"flow_dst_min_l4_payload_len":0,"flow_src_max_l4_payload_len":517,"flow_dst_max_l4_payload_len":1424,"flow_src_tot_l4_payload_len":517,"flow_dst_tot_l4_payload_len":1424,"midstream":0,"thread_ts_usec":1581113657863699,"l3_proto":"ip4","src_ip":"192.168.1.13","dst_ip":"140.82.114.4","src_port":55744,"dst_port":443,"l4_proto":"tcp","ndpi": {"confidence": {"6":"DPI"},"proto":"TLS.Github","proto_id":"91.203","encrypted":1,"breed":"Acceptable","category_id":15,"category":"Collaborative","hostname":"github.com","tls": {"version":"TLSv1.2","ja3":"2a26b1a62e40d25d4de3babc9d532f30","ja3s":"ae53107a2e47ea20c72ac44821a728bf","unsafe_cipher":0,"cipher":"TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256","alpn":"http\/1.1"}}}
 01417{"flow_event_id":8,"flow_event_name":"detection-update","thread_id":0,"packet_id":7,"source":"443-git.pcap","alias":"nDPId-test","flow_id":1,"flow_state":"info","flow_src_packets_processed":3,"flow_dst_packets_processed":4,"flow_first_seen":1581113657633853,"flow_src_last_pkt_time":1581113657751016,"flow_dst_last_pkt_time":1581113657863749,"flow_idle_time":3285032704,"flow_src_min_l4_payload_len":0,"flow_dst_min_l4_payload_len":0,"flow_src_max_l4_payload_len":517,"flow_dst_max_l4_payload_len":1424,"flow_src_tot_l4_payload_len":517,"flow_dst_tot_l4_payload_len":3550,"midstream":0,"thread_ts_usec":1581113657863749,"l3_proto":"ip4","src_ip":"192.168.1.13","dst_ip":"140.82.114.4","src_port":55744,"dst_port":443,"l4_proto":"tcp","ndpi": {"confidence": {"6":"DPI"},"proto":"TLS.Github","proto_id":"91.203","encrypted":1,"breed":"Acceptable","category_id":15,"category":"Collaborative","hostname":"github.com","tls": {"version":"TLSv1.2","server_names":"github.com,www.github.com","ja3":"2a26b1a62e40d25d4de3babc9d532f30","ja3s":"ae53107a2e47ea20c72ac44821a728bf","unsafe_cipher":0,"cipher":"TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256","issuerDN":"C=US, O=DigiCert Inc, OU=www.digicert.com, CN=DigiCert SHA2 Extended Validation Server CA","subjectDN":"C=US, ST=California, L=San Francisco, O=GitHub, Inc., CN=github.com","alpn":"http\/1.1","fingerprint":"CA:06:F5:6B:25:8B:7A:0D:4F:2B:05:47:09:39:47:86:51:15:19:84"}}}
-01697{"flow_event_id":5,"flow_event_name":"analyse","thread_id":0,"packet_id":32,"source":"443-git.pcap","alias":"nDPId-test","flow_id":1,"flow_state":"finished","flow_src_packets_processed":17,"flow_dst_packets_processed":15,"flow_first_seen":1581113657633853,"flow_src_last_pkt_time":1581113658139408,"flow_dst_last_pkt_time":1581113658139371,"flow_idle_time":3285032704,"flow_src_min_l4_payload_len":0,"flow_dst_min_l4_payload_len":0,"flow_src_max_l4_payload_len":517,"flow_dst_max_l4_payload_len":1424,"flow_src_tot_l4_payload_len":850,"flow_dst_tot_l4_payload_len":8277,"midstream":0,"thread_ts_usec":1581113658139408,"l3_proto":"ip4","src_ip":"192.168.1.13","dst_ip":"140.82.114.4","src_port":55744,"dst_port":443,"l4_proto":"tcp","flow_datalink":1,"flow_max_packets":3,"data_analysis": {"iat": {"min":2,"avg":32615.3,"max":143502,"stddev":53225.8,"var":2832981760.0,"ent":3.2,"data": [110467,110568,6595,119379,41,9,112809,2,11075,123994,112907,571,143502,5,142911,2,6496,2,14,6523,7,6,115,82,1242,13,1267,3,237,2,227]},"pktlen": {"min":66,"avg":351.8,"max":1490,"stddev":464.4,"var":215710.4,"ent":4.0,"data": [78,74,66,583,1490,1490,768,66,66,192,117,66,273,437,140,66,66,100,358,99,66,66,66,164,66,1465,622,66,66,1465,486,66]},"bins": {"c_to_s": [14,0,0,1,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"s_to_c": [1,3,1,1,0,0,0,0,0,1,0,1,0,1,0,0,0,1,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,2,0,0,0]},"directions": [0,1,0,0,1,1,1,0,0,0,1,0,0,1,1,0,0,1,1,1,0,0,0,1,0,1,1,0,0,1,1,0]},"ndpi": {"confidence": {"6":"DPI"},"proto":"TLS.Github","proto_id":"91.203","encrypted":1,"breed":"Acceptable","category_id":15,"category":"Collaborative"}}
+02095{"flow_event_id":5,"flow_event_name":"analyse","thread_id":0,"packet_id":32,"source":"443-git.pcap","alias":"nDPId-test","flow_id":1,"flow_state":"finished","flow_src_packets_processed":17,"flow_dst_packets_processed":15,"flow_first_seen":1581113657633853,"flow_src_last_pkt_time":1581113658139408,"flow_dst_last_pkt_time":1581113658139371,"flow_idle_time":3285032704,"flow_src_min_l4_payload_len":0,"flow_dst_min_l4_payload_len":0,"flow_src_max_l4_payload_len":517,"flow_dst_max_l4_payload_len":1424,"flow_src_tot_l4_payload_len":850,"flow_dst_tot_l4_payload_len":8277,"midstream":0,"thread_ts_usec":1581113658139408,"l3_proto":"ip4","src_ip":"192.168.1.13","dst_ip":"140.82.114.4","src_port":55744,"dst_port":443,"l4_proto":"tcp","flow_datalink":1,"flow_max_packets":3,"data_analysis": {"iat": {"min":2,"avg":32615.3,"max":143502,"stddev":53225.8,"var":2832981760.0,"ent":3.2,"data": [110467,110568,6595,119379,41,9,112809,2,11075,123994,112907,571,143502,5,142911,2,6496,2,14,6523,7,6,115,82,1242,13,1267,3,237,2,227]},"pktlen": {"min":52,"avg":337.8,"max":1476,"stddev":464.4,"var":215710.4,"ent":4.0,"data": [64,60,52,569,1476,1476,754,52,52,178,103,52,259,423,126,52,52,86,344,85,52,52,52,150,52,1451,608,52,52,1451,472,52]},"bins": {"c_to_s": [14,0,0,1,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"s_to_c": [1,3,1,1,0,0,0,0,0,1,0,1,0,1,0,0,0,1,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,2,0,0,0]},"directions": [0,1,0,0,1,1,1,0,0,0,1,0,0,1,1,0,0,1,1,1,0,0,0,1,0,1,1,0,0,1,1,0],"entropies": [4.341937065,5.174957275,4.831954479,4.223120689,6.954095364,7.397567272,7.645401001,5.014835358,4.976373672,6.355282307,5.929066658,4.937911987,6.952417850,7.419026852,6.223026752,4.937911987,4.976373672,5.637029648,7.370140076,5.726850986,4.937911987,4.937911987,4.899450302,6.443542957,4.976373672,7.866954327,7.624365330,5.014835358,5.014835358,7.857865334,7.532955170,5.014835358]},"ndpi": {"confidence": {"6":"DPI"},"proto":"TLS.Github","proto_id":"91.203","encrypted":1,"breed":"Acceptable","category_id":15,"category":"Collaborative"}}
 00925{"flow_event_id":2,"flow_event_name":"end","thread_id":0,"packet_id":70,"source":"443-git.pcap","alias":"nDPId-test","flow_id":1,"flow_state":"finished","flow_src_packets_processed":35,"flow_dst_packets_processed":35,"flow_first_seen":1581113657633853,"flow_src_last_pkt_time":1581113658456571,"flow_dst_last_pkt_time":1581113658456501,"flow_idle_time":3285032704,"flow_src_min_l4_payload_len":0,"flow_dst_min_l4_payload_len":0,"flow_src_max_l4_payload_len":517,"flow_dst_max_l4_payload_len":1424,"flow_src_tot_l4_payload_len":881,"flow_dst_tot_l4_payload_len":31704,"midstream":0,"thread_ts_usec":1581113658456571,"l3_proto":"ip4","src_ip":"192.168.1.13","dst_ip":"140.82.114.4","src_port":55744,"dst_port":443,"l4_proto":"tcp","flow_datalink":1,"flow_max_packets":3,"ndpi": {"confidence": {"6":"DPI"},"proto":"TLS.Github","proto_id":"91.203","encrypted":1,"breed":"Acceptable","category_id":15,"category":"Collaborative"}}
 00560{"daemon_event_id":3,"daemon_event_name":"shutdown","thread_id":0,"packet_id":70,"source":"443-git.pcap","alias":"nDPId-test","packets-captured":70,"packets-processed":70,"total-skipped-flows":0,"total-l4-payload-len":32585,"total-not-detected-flows":0,"total-guessed-flows":0,"total-detected-flows":1,"total-detection-updates":2,"total-updates":0,"current-active-flows":0,"total-active-flows":1,"total-idle-flows":1,"total-compressions":0,"total-compression-diff":0,"current-compression-diff":0,"total-events-serialized":12,"global_ts_usec":1581113658456571}
 ~~~~~~~~~~~~~~~~~~~~ SUMMARY ~~~~~~~~~~~~~~~~~~~~
@@ -18,10 +18,10 @@
 ~~ total active/idle flows...: 1/1
 ~~ total timeout flows.......: 0
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-~~ total memory allocated....: 6048145 bytes
-~~ total memory freed........: 6048145 bytes
-~~ total allocations/frees...: 121566/121566
+~~ total memory allocated....: 6048281 bytes
+~~ total memory freed........: 6048281 bytes
+~~ total allocations/frees...: 121567/121567
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 ~~ json string min len.......: 492 chars
-~~ json string max len.......: 1702 chars
-~~ json string avg len.......: 1066 chars
+~~ json string max len.......: 2100 chars
+~~ json string avg len.......: 1240 chars