Improved flown analyse event:

 * store packet directions
 * merged direction based IATs
 * merged direction based PKTLENs

Signed-off-by: Toni Uhlig <matzeton@googlemail.com>

1 files changed, 7 insertions, 7 deletions

diff --git a/test/results/ftp.pcap.out b/test/results/ftp.pcap.out
index 170360984..e979af4e0 100644
--- a/test/results/ftp.pcap.out
+++ b/test/results/ftp.pcap.out
@@ -5,7 +5,7 @@
 00532{"packet_event_id":2,"packet_event_name":"packet-flow","thread_id":0,"packet_id":2,"source":"ftp.pcap","alias":"nDPId-test","flow_id":1,"flow_packet_id":2,"flow_src_last_pkt_time":1552590234892296,"flow_dst_last_pkt_time":1552590234919708,"flow_idle_time":3285032704,"pkt_oversize":false,"pkt_caplen":74,"pkt_type":2048,"pkt_l3_offset":14,"pkt_l4_offset":34,"pkt_len":74,"pkt_l4_len":40,"thread_ts_usec":1552590234919708,"pkt":"xCwDBkn+EBMx8Tl2CABFAAA8AABAADYG4XRagkZJwKgB1AAVxgZYKsHSoyOX7qASqbA+KAAAAgQFrAQCCAoSZ\/tNO1eYmQEDAw4="}
 00520{"packet_event_id":2,"packet_event_name":"packet-flow","thread_id":0,"packet_id":3,"source":"ftp.pcap","alias":"nDPId-test","flow_id":1,"flow_packet_id":3,"flow_src_last_pkt_time":1552590234919816,"flow_dst_last_pkt_time":1552590234919708,"flow_idle_time":3285032704,"pkt_oversize":false,"pkt_caplen":66,"pkt_type":2048,"pkt_l3_offset":14,"pkt_l4_offset":34,"pkt_len":66,"pkt_l4_len":32,"thread_ts_usec":1552590234919816,"pkt":"EBMx8Tl2xCwDBkn+CABFAAA0AABAAEAGAADAqAHUWoJGScYGABWjI5fuWCrB04AQECxjbgAAAQEICjtXmLQSZ\/tN"}
 01043{"flow_event_id":7,"flow_event_name":"detected","thread_id":0,"packet_id":12,"source":"ftp.pcap","alias":"nDPId-test","flow_id":1,"flow_state":"info","flow_src_packets_processed":6,"flow_dst_packets_processed":6,"flow_first_seen":1552590234892296,"flow_src_last_pkt_time":1552590234976972,"flow_dst_last_pkt_time":1552590235066945,"flow_idle_time":3285032704,"flow_src_min_l4_payload_len":0,"flow_dst_min_l4_payload_len":0,"flow_src_max_l4_payload_len":16,"flow_dst_max_l4_payload_len":34,"flow_src_tot_l4_payload_len":29,"flow_dst_tot_l4_payload_len":77,"midstream":0,"thread_ts_usec":1552590235066945,"l3_proto":"ip4","src_ip":"192.168.1.212","dst_ip":"90.130.70.73","src_port":50694,"dst_port":21,"l4_proto":"tcp","ndpi": {"flow_risk": {"22": {"risk":"Unsafe Protocol","severity":"Low","risk_score": {"total":750,"client":575,"server":175}}},"confidence": {"6":"DPI"},"proto":"FTP_CONTROL","proto_id":"1","encrypted":0,"breed":"Unsafe","category_id":7,"category":"Download","ftp": {"user":"anonymous","password":"NcFTP@","auth_failed":0}}}
-01665{"flow_event_id":5,"flow_event_name":"analyse","thread_id":0,"packet_id":32,"source":"ftp.pcap","alias":"nDPId-test","flow_id":1,"flow_state":"finished","flow_src_packets_processed":18,"flow_dst_packets_processed":14,"flow_first_seen":1552590234892296,"flow_src_last_pkt_time":1552590235175924,"flow_dst_last_pkt_time":1552590235202548,"flow_idle_time":3285032704,"flow_src_min_l4_payload_len":0,"flow_dst_min_l4_payload_len":0,"flow_src_max_l4_payload_len":30,"flow_dst_max_l4_payload_len":241,"flow_src_tot_l4_payload_len":86,"flow_dst_tot_l4_payload_len":532,"midstream":0,"thread_ts_usec":1552590235202548,"l3_proto":"ip4","src_ip":"192.168.1.212","dst_ip":"90.130.70.73","src_port":50694,"dst_port":21,"l4_proto":"tcp","flow_datalink":1,"flow_max_packets":3,"data_analysis": {"iat": {"flow_min":6,"flow_avg":19157.4,"flow_max":90047,"flow_stddev":20644.4,"c_to_s_min":6,"c_to_s_avg":16684.0,"c_to_s_max":90047,"c_to_s_stddev":22507.5,"s_to_c_min":21,"s_to_c_avg":22160.9,"s_to_c_max":69061,"s_to_c_stddev":17666.8},"pktlen": {"c_to_s_min":66,"c_to_s_avg":71.4,"c_to_s_max":96,"c_to_s_stddev":8.0,"s_to_c_min":66,"s_to_c_avg":104.6,"s_to_c_max":307,"s_to_c_stddev":58.9},"bins": {"c_to_s": [18,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"s_to_c": [8,4,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]}},"ndpi": {"flow_risk": {"22": {"risk":"Unsafe Protocol","severity":"Low","risk_score": {"total":750,"client":575,"server":175}}},"confidence": {"6":"DPI"},"proto":"FTP_CONTROL","proto_id":"1","encrypted":0,"breed":"Unsafe","category_id":7,"category":"Download"}}
+01795{"flow_event_id":5,"flow_event_name":"analyse","thread_id":0,"packet_id":32,"source":"ftp.pcap","alias":"nDPId-test","flow_id":1,"flow_state":"finished","flow_src_packets_processed":18,"flow_dst_packets_processed":14,"flow_first_seen":1552590234892296,"flow_src_last_pkt_time":1552590235175924,"flow_dst_last_pkt_time":1552590235202548,"flow_idle_time":3285032704,"flow_src_min_l4_payload_len":0,"flow_dst_min_l4_payload_len":0,"flow_src_max_l4_payload_len":30,"flow_dst_max_l4_payload_len":241,"flow_src_tot_l4_payload_len":86,"flow_dst_tot_l4_payload_len":532,"midstream":0,"thread_ts_usec":1552590235202548,"l3_proto":"ip4","src_ip":"192.168.1.212","dst_ip":"90.130.70.73","src_port":50694,"dst_port":21,"l4_proto":"tcp","flow_datalink":1,"flow_max_packets":3,"data_analysis": {"iat": {"min":6,"avg":19157.4,"max":90047,"stddev":20644.4,"var":426190272.0,"ent":4.1,"data": [27412,27520,29008,29012,526,27660,315,27401,217,69061,21193,90047,306,27070,21,26780,133,26972,64,26857,6,275,27478,27261,90,29,651,27147,26517,90,26761,0]},"pktlen": {"min":66,"avg":85.9,"max":307,"stddev":42.7,"var":1824.0,"ent":4.9,"data": [78,74,66,86,66,82,66,100,66,79,66,89,66,71,66,100,66,72,81,131,66,66,77,110,66,307,66,96,88,66,71,100]},"bins": {"c_to_s": [18,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"s_to_c": [8,4,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]},"directions": [0,1,0,1,0,0,1,1,0,0,1,1,0,0,1,1,0,0,1,1,0,0,0,1,0,1,0,0,1,0,0,1]},"ndpi": {"flow_risk": {"22": {"risk":"Unsafe Protocol","severity":"Low","risk_score": {"total":750,"client":575,"server":175}}},"confidence": {"6":"DPI"},"proto":"FTP_CONTROL","proto_id":"1","encrypted":0,"breed":"Unsafe","category_id":7,"category":"Download"}}
 00751{"flow_event_id":1,"flow_event_name":"new","thread_id":0,"packet_id":37,"source":"ftp.pcap","alias":"nDPId-test","flow_id":2,"flow_state":"info","flow_src_packets_processed":1,"flow_dst_packets_processed":0,"flow_first_seen":1552590236580045,"flow_src_last_pkt_time":1552590236580045,"flow_dst_last_pkt_time":1552590236580045,"flow_idle_time":3285032704,"flow_src_min_l4_payload_len":0,"flow_dst_min_l4_payload_len":0,"flow_src_max_l4_payload_len":0,"flow_dst_max_l4_payload_len":0,"flow_src_tot_l4_payload_len":0,"flow_dst_tot_l4_payload_len":0,"midstream":0,"thread_ts_usec":1552590236580045,"l3_proto":"ip4","src_ip":"192.168.1.212","dst_ip":"90.130.70.73","src_port":50695,"dst_port":25685,"l4_proto":"tcp","flow_datalink":1,"flow_max_packets":3}
 00538{"packet_event_id":2,"packet_event_name":"packet-flow","thread_id":0,"packet_id":37,"source":"ftp.pcap","alias":"nDPId-test","flow_id":2,"flow_packet_id":1,"flow_src_last_pkt_time":1552590236580045,"flow_dst_last_pkt_time":1552590236580045,"flow_idle_time":3285032704,"pkt_oversize":false,"pkt_caplen":78,"pkt_type":2048,"pkt_l3_offset":14,"pkt_l4_offset":34,"pkt_len":78,"pkt_l4_len":44,"thread_ts_usec":1552590236580045,"pkt":"EBMx8Tl2xCwDBkn+CABFAABAAABAAEAGAADAqAHUWoJGScYHZFXuwKKMAAAAALAC\/\/9jegAAAgQFtAEDAwUBAQgKO1efIQAAAAAEAgAA"}
 00533{"packet_event_id":2,"packet_event_name":"packet-flow","thread_id":0,"packet_id":38,"source":"ftp.pcap","alias":"nDPId-test","flow_id":2,"flow_packet_id":2,"flow_src_last_pkt_time":1552590236580045,"flow_dst_last_pkt_time":1552590236608252,"flow_idle_time":3285032704,"pkt_oversize":false,"pkt_caplen":74,"pkt_type":2048,"pkt_l3_offset":14,"pkt_l4_offset":34,"pkt_len":74,"pkt_l4_len":40,"thread_ts_usec":1552590236608252,"pkt":"xCwDBkn+EBMx8Tl2CABFAAA8AABAADYG4XRagkZJwKgB1GRVxgdmK2Nw7sCijaASqbDL3QAAAgQFrAQCCAoSZ\/zzO1efIQEDAw4="}
@@ -15,7 +15,7 @@
 00538{"packet_event_id":2,"packet_event_name":"packet-flow","thread_id":0,"packet_id":66,"source":"ftp.pcap","alias":"nDPId-test","flow_id":3,"flow_packet_id":1,"flow_src_last_pkt_time":1552590241545143,"flow_dst_last_pkt_time":1552590241545143,"flow_idle_time":3285032704,"pkt_oversize":false,"pkt_caplen":78,"pkt_type":2048,"pkt_l3_offset":14,"pkt_l4_offset":34,"pkt_len":78,"pkt_l4_len":44,"thread_ts_usec":1552590241545143,"pkt":"EBMx8Tl2xCwDBkn+CABFAABAAABAAEAGAADAqAHUWoJGScYIX8sNBxpOAAAAALAC\/\/9jegAAAgQFtAEDAwUBAQgKO1eyYgAAAAAEAgAA"}
 00533{"packet_event_id":2,"packet_event_name":"packet-flow","thread_id":0,"packet_id":67,"source":"ftp.pcap","alias":"nDPId-test","flow_id":3,"flow_packet_id":2,"flow_src_last_pkt_time":1552590241545143,"flow_dst_last_pkt_time":1552590241573913,"flow_idle_time":3285032704,"pkt_oversize":false,"pkt_caplen":74,"pkt_type":2048,"pkt_l3_offset":14,"pkt_l4_offset":34,"pkt_len":74,"pkt_l4_len":40,"thread_ts_usec":1552590241573913,"pkt":"xCwDBkn+EBMx8Tl2CABFAAA8AABAADYG4XRagkZJwKgB1F\/LxggMTnkwDQcaT6ASqbBmYgAAAgQFrAQCCAoSaAHMO1eyYgEDAw4="}
 00520{"packet_event_id":2,"packet_event_name":"packet-flow","thread_id":0,"packet_id":68,"source":"ftp.pcap","alias":"nDPId-test","flow_id":3,"flow_packet_id":3,"flow_src_last_pkt_time":1552590241573957,"flow_dst_last_pkt_time":1552590241573913,"flow_idle_time":3285032704,"pkt_oversize":false,"pkt_caplen":66,"pkt_type":2048,"pkt_l3_offset":14,"pkt_l4_offset":34,"pkt_len":66,"pkt_l4_len":32,"thread_ts_usec":1552590241573957,"pkt":"EBMx8Tl2xCwDBkn+CABFAAA0AABAAEAGAADAqAHUWoJGScYIX8sNBxpPDE55MYAQECxjbgAAAQEICjtXsn0SaAHM"}
-01406{"flow_event_id":5,"flow_event_name":"analyse","thread_id":0,"packet_id":100,"source":"ftp.pcap","alias":"nDPId-test","flow_id":3,"flow_state":"info","flow_src_packets_processed":13,"flow_dst_packets_processed":19,"flow_first_seen":1552590241545143,"flow_src_last_pkt_time":1552590241637688,"flow_dst_last_pkt_time":1552590241639633,"flow_idle_time":3285032704,"flow_src_min_l4_payload_len":0,"flow_dst_min_l4_payload_len":0,"flow_src_max_l4_payload_len":0,"flow_dst_max_l4_payload_len":1440,"flow_src_tot_l4_payload_len":0,"flow_dst_tot_l4_payload_len":24480,"midstream":0,"thread_ts_usec":1552590241639633,"l3_proto":"ip4","src_ip":"192.168.1.212","dst_ip":"90.130.70.73","src_port":50696,"dst_port":24523,"l4_proto":"tcp","flow_datalink":1,"flow_max_packets":3,"data_analysis": {"iat": {"flow_min":2,"flow_avg":6033.4,"flow_max":29579,"flow_stddev":11108.9,"c_to_s_min":43,"c_to_s_avg":7712.1,"c_to_s_max":29566,"c_to_s_stddev":12144.9,"s_to_c_min":2,"s_to_c_avg":4973.2,"s_to_c_max":29579,"s_to_c_stddev":10261.0},"pktlen": {"c_to_s_min":66,"c_to_s_avg":67.8,"c_to_s_max":78,"c_to_s_stddev":4.3,"s_to_c_min":66,"s_to_c_avg":1354.8,"s_to_c_max":1506,"s_to_c_stddev":440.7},"bins": {"c_to_s": [13,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"s_to_c": [2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,17,0,0]}}}
+01550{"flow_event_id":5,"flow_event_name":"analyse","thread_id":0,"packet_id":100,"source":"ftp.pcap","alias":"nDPId-test","flow_id":3,"flow_state":"info","flow_src_packets_processed":13,"flow_dst_packets_processed":19,"flow_first_seen":1552590241545143,"flow_src_last_pkt_time":1552590241637688,"flow_dst_last_pkt_time":1552590241639633,"flow_idle_time":3285032704,"flow_src_min_l4_payload_len":0,"flow_dst_min_l4_payload_len":0,"flow_src_max_l4_payload_len":0,"flow_dst_max_l4_payload_len":1440,"flow_src_tot_l4_payload_len":0,"flow_dst_tot_l4_payload_len":24480,"midstream":0,"thread_ts_usec":1552590241639633,"l3_proto":"ip4","src_ip":"192.168.1.212","dst_ip":"90.130.70.73","src_port":50696,"dst_port":24523,"l4_proto":"tcp","flow_datalink":1,"flow_max_packets":3,"data_analysis": {"iat": {"min":2,"avg":6033.4,"max":29579,"stddev":11108.9,"var":123407192.0,"ent":3.1,"data": [28770,28814,29579,29566,281,284,597,608,340,458,790,363,375,64,327,2,379,43,300,27513,27767,195,211,1702,115,4,1805,1866,1903,218,1796,0]},"pktlen": {"min":66,"avg":832.0,"max":1506,"stddev":717.5,"var":514855.0,"ent":4.3,"data": [78,74,66,1506,78,1506,66,1506,66,1506,1506,66,1506,66,1506,1506,1506,66,66,1506,1506,66,1506,66,1506,1506,66,66,1506,66,1506,1506]},"bins": {"c_to_s": [13,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"s_to_c": [2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,17,0,0]},"directions": [0,1,0,1,0,1,0,1,0,1,1,0,1,0,1,1,1,0,0,1,1,0,1,0,1,1,1,0,1,0,1,1]}}
 00806{"flow_event_id":9,"flow_event_name":"not-detected","thread_id":0,"packet_id":100,"source":"ftp.pcap","alias":"nDPId-test","flow_id":3,"flow_state":"info","flow_src_packets_processed":13,"flow_dst_packets_processed":19,"flow_first_seen":1552590241545143,"flow_src_last_pkt_time":1552590241637688,"flow_dst_last_pkt_time":1552590241639633,"flow_idle_time":3285032704,"flow_src_min_l4_payload_len":0,"flow_dst_min_l4_payload_len":0,"flow_src_max_l4_payload_len":0,"flow_dst_max_l4_payload_len":1440,"flow_src_tot_l4_payload_len":0,"flow_dst_tot_l4_payload_len":24480,"midstream":0,"thread_ts_usec":1552590241639633,"l3_proto":"ip4","src_ip":"192.168.1.212","dst_ip":"90.130.70.73","src_port":50696,"dst_port":24523,"l4_proto":"tcp","ndpi": {"proto":"Unknown","proto_id":"0","encrypted":0,"breed":"Unrated"}}
 00845{"flow_event_id":2,"flow_event_name":"end","thread_id":0,"packet_id":1192,"source":"ftp.pcap","alias":"nDPId-test","flow_id":3,"flow_state":"finished","flow_src_packets_processed":380,"flow_dst_packets_processed":735,"flow_first_seen":1552590241545143,"flow_src_last_pkt_time":1552590241851108,"flow_dst_last_pkt_time":1552590241878454,"flow_idle_time":3285032704,"flow_src_min_l4_payload_len":0,"flow_dst_min_l4_payload_len":0,"flow_src_max_l4_payload_len":0,"flow_dst_max_l4_payload_len":1440,"flow_src_tot_l4_payload_len":0,"flow_dst_tot_l4_payload_len":1048576,"midstream":0,"thread_ts_usec":1552590243371057,"l3_proto":"ip4","src_ip":"192.168.1.212","dst_ip":"90.130.70.73","src_port":50696,"dst_port":24523,"l4_proto":"tcp","flow_datalink":1,"flow_max_packets":3,"ndpi": {"proto":"Unknown","proto_id":"0","encrypted":0,"breed":"Unrated"}}
 01024{"flow_event_id":2,"flow_event_name":"end","thread_id":0,"packet_id":1192,"source":"ftp.pcap","alias":"nDPId-test","flow_id":1,"flow_state":"finished","flow_src_packets_processed":41,"flow_dst_packets_processed":27,"flow_first_seen":1552590234892296,"flow_src_last_pkt_time":1552590243340268,"flow_dst_last_pkt_time":1552590243371057,"flow_idle_time":3285032704,"flow_src_min_l4_payload_len":0,"flow_dst_min_l4_payload_len":0,"flow_src_max_l4_payload_len":30,"flow_dst_max_l4_payload_len":241,"flow_src_tot_l4_payload_len":174,"flow_dst_tot_l4_payload_len":889,"midstream":0,"thread_ts_usec":1552590243371057,"l3_proto":"ip4","src_ip":"192.168.1.212","dst_ip":"90.130.70.73","src_port":50694,"dst_port":21,"l4_proto":"tcp","flow_datalink":1,"flow_max_packets":3,"ndpi": {"flow_risk": {"22": {"risk":"Unsafe Protocol","severity":"Low","risk_score": {"total":750,"client":575,"server":175}}},"confidence": {"6":"DPI"},"proto":"FTP_CONTROL","proto_id":"1","encrypted":0,"breed":"Unsafe","category_id":7,"category":"Download"}}
@@ -29,10 +29,10 @@
 ~~ total active/idle flows...: 3/3
 ~~ total timeout flows.......: 0
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-~~ total memory allocated....: 6081036 bytes
-~~ total memory freed........: 6081036 bytes
-~~ total allocations/frees...: 122709/122709
+~~ total memory allocated....: 6079644 bytes
+~~ total memory freed........: 6079644 bytes
+~~ total allocations/frees...: 122703/122703
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 ~~ json string min len.......: 488 chars
-~~ json string max len.......: 1670 chars
-~~ json string avg len.......: 1078 chars
+~~ json string max len.......: 1800 chars
+~~ json string avg len.......: 1143 chars