Improved py-machine-learning example.

 * c-analysed: fixed quoting bug
 * nDPId: fixed invalid iat storing/serialisation
 * nDPId: free data analysis after event was sent

Signed-off-by: Toni Uhlig <matzeton@googlemail.com>
Signed-off-by: lns <matzeton@googlemail.com>

1 files changed, 3 insertions, 3 deletions

diff --git a/test/results/dns_doh.pcap.out b/test/results/dns_doh.pcap.out
index 56603dbcc..34b00427e 100644
--- a/test/results/dns_doh.pcap.out
+++ b/test/results/dns_doh.pcap.out
@@ -6,7 +6,7 @@
 00507{"packet_event_id":2,"packet_event_name":"packet-flow","thread_id":0,"packet_id":3,"source":"dns_doh.pcap","alias":"nDPId-test","flow_id":1,"flow_packet_id":3,"flow_src_last_pkt_time":1571089200876498,"flow_dst_last_pkt_time":1571089200876406,"flow_idle_time":3285032704,"pkt_oversize":false,"pkt_caplen":54,"pkt_type":2048,"pkt_l3_offset":14,"pkt_l4_offset":34,"pkt_len":54,"pkt_l4_len":20,"thread_ts_usec":1571089200876498,"pkt":"WkBO7NFkeDHBvV4kCABFAAAoAABAAEAGI66sFAoEaBD4+cLVAbuk7FgjymHcL1AQEAAggAAA"}
 01118{"flow_event_id":7,"flow_event_name":"detected","thread_id":0,"packet_id":4,"source":"dns_doh.pcap","alias":"nDPId-test","flow_id":1,"flow_state":"info","flow_src_packets_processed":3,"flow_dst_packets_processed":1,"flow_first_seen":1571089200789290,"flow_src_last_pkt_time":1571089200878306,"flow_dst_last_pkt_time":1571089200876406,"flow_idle_time":3285032704,"flow_src_min_l4_payload_len":0,"flow_dst_min_l4_payload_len":0,"flow_src_max_l4_payload_len":517,"flow_dst_max_l4_payload_len":0,"flow_src_tot_l4_payload_len":517,"flow_dst_tot_l4_payload_len":0,"midstream":0,"thread_ts_usec":1571089200878306,"l3_proto":"ip4","src_ip":"172.20.10.4","dst_ip":"104.16.248.249","src_port":49877,"dst_port":443,"l4_proto":"tcp","ndpi": {"confidence": {"6":"DPI"},"proto":"TLS.DoH_DoT","proto_id":"91.196","encrypted":1,"breed":"Fun","category_id":14,"category":"Network","hostname":"mozilla.cloudflare-dns.com","tls": {"version":"TLSv1.2","ja3":"b20b44b18b853ef29ab773e921b03422","ja3s":"","unsafe_cipher":0,"cipher":"TLS_NULL_WITH_NULL_NULL","alpn":"h2,http\/1.1","tls_supported_versions":"TLSv1.3,TLSv1.2,TLSv1.1,TLSv1"}}}
 01163{"flow_event_id":8,"flow_event_name":"detection-update","thread_id":0,"packet_id":6,"source":"dns_doh.pcap","alias":"nDPId-test","flow_id":1,"flow_state":"info","flow_src_packets_processed":3,"flow_dst_packets_processed":3,"flow_first_seen":1571089200789290,"flow_src_last_pkt_time":1571089200878306,"flow_dst_last_pkt_time":1571089200968629,"flow_idle_time":3285032704,"flow_src_min_l4_payload_len":0,"flow_dst_min_l4_payload_len":0,"flow_src_max_l4_payload_len":517,"flow_dst_max_l4_payload_len":1300,"flow_src_tot_l4_payload_len":517,"flow_dst_tot_l4_payload_len":1300,"midstream":0,"thread_ts_usec":1571089200968629,"l3_proto":"ip4","src_ip":"172.20.10.4","dst_ip":"104.16.248.249","src_port":49877,"dst_port":443,"l4_proto":"tcp","ndpi": {"confidence": {"6":"DPI"},"proto":"TLS.DoH_DoT","proto_id":"91.196","encrypted":1,"breed":"Fun","category_id":14,"category":"Network","hostname":"mozilla.cloudflare-dns.com","tls": {"version":"TLSv1.3","ja3":"b20b44b18b853ef29ab773e921b03422","ja3s":"eb1d94daa7e0344597e756a1fb6e7054","unsafe_cipher":0,"cipher":"TLS_AES_128_GCM_SHA256","alpn":"h2,http\/1.1","tls_supported_versions":"TLSv1.3,TLSv1.2,TLSv1.1,TLSv1"}}}
-02088{"flow_event_id":5,"flow_event_name":"analyse","thread_id":0,"packet_id":32,"source":"dns_doh.pcap","alias":"nDPId-test","flow_id":1,"flow_state":"finished","flow_src_packets_processed":18,"flow_dst_packets_processed":14,"flow_first_seen":1571089200789290,"flow_src_last_pkt_time":1571089201723583,"flow_dst_last_pkt_time":1571089201764372,"flow_idle_time":3285032704,"flow_src_min_l4_payload_len":0,"flow_dst_min_l4_payload_len":0,"flow_src_max_l4_payload_len":517,"flow_dst_max_l4_payload_len":1300,"flow_src_tot_l4_payload_len":1424,"flow_dst_tot_l4_payload_len":4202,"midstream":0,"thread_ts_usec":1571089201764372,"l3_proto":"ip4","src_ip":"172.20.10.4","dst_ip":"104.16.248.249","src_port":49877,"dst_port":443,"l4_proto":"tcp","flow_datalink":1,"flow_max_packets":3,"data_analysis": {"iat": {"min":1,"avg":63645.8,"max":535341,"stddev":131829.5,"var":17379012608.0,"ent":3.0,"data": [87116,87208,1808,92218,5,2,90426,511,1485,930,26074,858,110,91,102733,7825,6,1,83431,1,17900,147557,535341,708,88830,66,525420,6,10702,6]},"pktlen": {"min":40,"avg":216.9,"max":1340,"stddev":327.3,"var":107137.2,"ent":3.9,"data": [64,52,40,557,40,1340,1340,40,40,489,40,104,210,283,119,40,577,390,71,40,40,40,71,40,102,133,102,143,40,40,244,71]},"bins": {"c_to_s": [9,2,3,1,0,1,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"s_to_c": [8,0,0,0,0,0,1,0,0,0,1,0,0,0,1,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0]},"directions": [0,1,0,0,1,1,1,0,0,1,0,0,0,0,0,1,1,1,1,0,0,0,0,1,0,0,0,0,1,1,1,1],"entropies": [4.441382408,4.801308632,4.503056526,5.369568825,4.730641365,7.827131748,7.862888336,4.630641460,4.453056335,7.522860050,4.630641460,5.744826317,6.939166546,7.200489998,6.276752949,4.730641365,7.589616776,7.428659439,5.699038506,4.730641365,4.730641365,4.680641174,5.688406467,4.780641556,6.111449242,6.391828060,6.039783001,6.407779217,4.780641556,4.730641365,7.064774990,5.558194637]},"ndpi": {"confidence": {"6":"DPI"},"proto":"TLS.DoH_DoT","proto_id":"91.196","encrypted":1,"breed":"Fun","category_id":14,"category":"Network"}}
+02090{"flow_event_id":5,"flow_event_name":"analyse","thread_id":0,"packet_id":32,"source":"dns_doh.pcap","alias":"nDPId-test","flow_id":1,"flow_state":"finished","flow_src_packets_processed":18,"flow_dst_packets_processed":14,"flow_first_seen":1571089200789290,"flow_src_last_pkt_time":1571089201723583,"flow_dst_last_pkt_time":1571089201764372,"flow_idle_time":3285032704,"flow_src_min_l4_payload_len":0,"flow_dst_min_l4_payload_len":0,"flow_src_max_l4_payload_len":517,"flow_dst_max_l4_payload_len":1300,"flow_src_tot_l4_payload_len":1424,"flow_dst_tot_l4_payload_len":4202,"midstream":0,"thread_ts_usec":1571089201764372,"l3_proto":"ip4","src_ip":"172.20.10.4","dst_ip":"104.16.248.249","src_port":49877,"dst_port":443,"l4_proto":"tcp","flow_datalink":1,"flow_max_packets":3,"data_analysis": {"iat": {"min":0,"avg":61592.7,"max":535341,"stddev":130172.4,"var":16944855040.0,"ent":3.0,"data": [87116,87208,1808,92218,5,2,90426,511,1485,930,26074,858,110,91,102733,7825,6,1,83431,1,0,17900,147557,535341,708,88830,66,525420,6,10702,6]},"pktlen": {"min":40,"avg":216.9,"max":1340,"stddev":327.3,"var":107137.2,"ent":3.9,"data": [64,52,40,557,40,1340,1340,40,40,489,40,104,210,283,119,40,577,390,71,40,40,40,71,40,102,133,102,143,40,40,244,71]},"bins": {"c_to_s": [9,2,3,1,0,1,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"s_to_c": [8,0,0,0,0,0,1,0,0,0,1,0,0,0,1,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0]},"directions": [0,1,0,0,1,1,1,0,0,1,0,0,0,0,0,1,1,1,1,0,0,0,0,1,0,0,0,0,1,1,1,1],"entropies": [4.441382408,4.801308632,4.503056526,5.369568825,4.730641365,7.827131748,7.862888336,4.630641460,4.453056335,7.522860050,4.630641460,5.744826317,6.939166546,7.200489998,6.276752949,4.730641365,7.589616776,7.428659439,5.699038506,4.730641365,4.730641365,4.680641174,5.688406467,4.780641556,6.111449242,6.391828060,6.039783001,6.407779217,4.780641556,4.730641365,7.064774990,5.558194637]},"ndpi": {"confidence": {"6":"DPI"},"proto":"TLS.DoH_DoT","proto_id":"91.196","encrypted":1,"breed":"Fun","category_id":14,"category":"Network"}}
 00916{"flow_event_id":3,"flow_event_name":"idle","thread_id":0,"packet_id":142,"source":"dns_doh.pcap","alias":"nDPId-test","flow_id":1,"flow_state":"finished","flow_src_packets_processed":86,"flow_dst_packets_processed":56,"flow_first_seen":1571089200789290,"flow_src_last_pkt_time":1571089204031014,"flow_dst_last_pkt_time":1571089204030791,"flow_idle_time":3285032704,"flow_src_min_l4_payload_len":0,"flow_dst_min_l4_payload_len":0,"flow_src_max_l4_payload_len":517,"flow_dst_max_l4_payload_len":1300,"flow_src_tot_l4_payload_len":3792,"flow_dst_tot_l4_payload_len":8866,"midstream":0,"thread_ts_usec":1571089204031014,"l3_proto":"ip4","src_ip":"172.20.10.4","dst_ip":"104.16.248.249","src_port":49877,"dst_port":443,"l4_proto":"tcp","flow_datalink":1,"flow_max_packets":3,"ndpi": {"confidence": {"6":"DPI"},"proto":"TLS.DoH_DoT","proto_id":"91.196","encrypted":1,"breed":"Fun","category_id":14,"category":"Network"}}
 00563{"daemon_event_id":3,"daemon_event_name":"shutdown","thread_id":0,"packet_id":142,"source":"dns_doh.pcap","alias":"nDPId-test","packets-captured":142,"packets-processed":142,"total-skipped-flows":0,"total-l4-payload-len":12658,"total-not-detected-flows":0,"total-guessed-flows":0,"total-detected-flows":1,"total-detection-updates":1,"total-updates":0,"current-active-flows":0,"total-active-flows":1,"total-idle-flows":1,"total-compressions":0,"total-compression-diff":0,"current-compression-diff":0,"total-events-serialized":11,"global_ts_usec":1571089204031014}
 ~~~~~~~~~~~~~~~~~~~~ SUMMARY ~~~~~~~~~~~~~~~~~~~~
@@ -22,5 +22,5 @@
 ~~ total allocations/frees...: 121636/121636
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 ~~ json string min len.......: 492 chars
-~~ json string max len.......: 2093 chars
-~~ json string avg len.......: 1226 chars
+~~ json string max len.......: 2095 chars
+~~ json string avg len.......: 1227 chars