Improved py-flow-info to provide more optional information about received timestamps.

 * py-flow-dashboard: Added color mapping for PieCharts/Graph that make more sense
 * nDPId: Renamed `flow_type' to a more precisely `flow_state'
 * nDPId: Changed the default setting to process only as much packets as libnDPI does

Signed-off-by: Toni Uhlig <matzeton@googlemail.com>

1 files changed, 7 insertions, 6 deletions

diff --git a/test/results/pinterest.pcap.out b/test/results/pinterest.pcap.out
index 26dddcc7d..b9d22f9df 100644
--- a/test/results/pinterest.pcap.out
+++ b/test/results/pinterest.pcap.out
@@ -1,4 +1,4 @@
-00442{"daemon_event_id":1,"daemon_event_name":"init","thread_id":0,"packet_id":0,"source":"pinterest.pcap","alias":"nDPId-test","max-flows-per-thread":2048,"max-idle-flows-per-thread":64,"tick-resolution":1000,"reader-thread-count":1,"flow-scan-interval":10000,"generic-max-idle-time":600000,"icmp-max-idle-time":120000,"udp-max-idle-time":180000,"tcp-max-idle-time":7560000,"max-packets-per-flow-to-send":3,"max-packets-per-flow-to-process":255}
+00441{"daemon_event_id":1,"daemon_event_name":"init","thread_id":0,"packet_id":0,"source":"pinterest.pcap","alias":"nDPId-test","max-flows-per-thread":2048,"max-idle-flows-per-thread":64,"tick-resolution":1000,"reader-thread-count":1,"flow-scan-interval":10000,"generic-max-idle-time":600000,"icmp-max-idle-time":120000,"udp-max-idle-time":180000,"tcp-max-idle-time":7560000,"max-packets-per-flow-to-send":3,"max-packets-per-flow-to-process":32}
 00602{"flow_event_id":1,"flow_event_name":"new","thread_id":0,"packet_id":1,"source":"pinterest.pcap","alias":"nDPId-test","flow_id":1,"flow_state":"info","flow_packets_processed":1,"flow_first_seen":1605289710318,"flow_last_seen":1605289710318,"flow_idle_time":7440000,"flow_min_l4_payload_len":0,"flow_max_l4_payload_len":0,"flow_tot_l4_payload_len":0,"flow_avg_l4_payload_len":0,"midstream":1,"ts_msec":1605289710318,"l3_proto":"ip6","src_ip":"2a01:cb01:2049:8b07:991d:ec85:28df:f629","dst_ip":"64:ff9b::9765:7854","src_port":33164,"dst_port":443,"l4_proto":"tcp","flow_datalink":1,"flow_max_packets":3}
 00488{"packet_event_id":2,"packet_event_name":"packet-flow","thread_id":0,"packet_id":1,"source":"pinterest.pcap","alias":"nDPId-test","flow_id":1,"flow_packet_id":1,"flow_last_seen":1605289710318,"flow_idle_time":7440000,"pkt_oversize":false,"pkt_caplen":86,"pkt_type":34525,"pkt_l3_offset":14,"pkt_l4_offset":54,"pkt_len":86,"pkt_l4_len":32,"ts_msec":1605289710318,"pkt":"qtsDr8lk5EKm5WPyht1gCMmjACAGQCoBywEgSYsHmR3shSjf9ikAZP+bAAAAAAAAAACXZXhUgYwBu9VDYL21LWgegBAB9TESAAABAQgKz6ojDMK4Yvg="}
 00490{"packet_event_id":2,"packet_event_name":"packet-flow","thread_id":0,"packet_id":2,"source":"pinterest.pcap","alias":"nDPId-test","flow_id":1,"flow_packet_id":2,"flow_last_seen":1605289710576,"flow_idle_time":7440000,"pkt_oversize":false,"pkt_caplen":86,"pkt_type":34525,"pkt_l3_offset":14,"pkt_l4_offset":54,"pkt_len":86,"pkt_l4_len":32,"ts_msec":1605289710576,"pkt":"5EKm5WPyqtsDr8lkht1gAAAAACAGPQBk\/5sAAAAAAAAAAJdleFQqAcsBIEmLB5kd7IUo3\/YpAbuBjLUtaB7VQ2C+gBALgY8wAAABAQgKwrkTpM+oCrY="}
@@ -123,6 +123,8 @@
 00714{"packet_event_id":2,"packet_event_name":"packet-flow","thread_id":0,"packet_id":9522,"source":"pinterest.pcap","alias":"nDPId-test","flow_id":22,"flow_packet_id":1,"flow_last_seen":1605289716168,"flow_idle_time":7440000,"pkt_oversize":false,"pkt_caplen":244,"pkt_type":34525,"pkt_l3_offset":14,"pkt_l4_offset":54,"pkt_len":244,"pkt_l4_len":190,"ts_msec":1605289716168,"pkt":"qtsDr8lk5EKm5WPyht1gB32\/AL4GQCoBywEgSYsHmR3shSjf9ikqABRQQAcIBQAAAAAAACADqioBu9lam\/a\/4e68gBgE1TyJAAABAQgKZPSVcMK4jAQXAwMAmbA2YtBqXOwsPZhf0xplQUhs5uebiQ6HrXX0rQcB3CzDNqt6KEFEtOrnLbiyKoAl0\/PfpLU5lSyfN4b6GWAPMuxRzKK1mYHeU6cm19ssJsGj28uoKpDNJuLbc68jHie5jcE8\/swMHjb\/rsshDlUuBkbS0PBg+fBq\/uDg8aBU7dQCoscpqfDhz7OaLw8PBcid6Woaoneonk0XRQ=="}
 00663{"packet_event_id":2,"packet_event_name":"packet-flow","thread_id":0,"packet_id":9523,"source":"pinterest.pcap","alias":"nDPId-test","flow_id":22,"flow_packet_id":2,"flow_last_seen":1605289716168,"flow_idle_time":7440000,"pkt_oversize":false,"pkt_caplen":209,"pkt_type":34525,"pkt_l3_offset":14,"pkt_l4_offset":54,"pkt_len":209,"pkt_l4_len":155,"ts_msec":1605289716168,"pkt":"qtsDr8lk5EKm5WPyht1gB32\/AJsGQCoBywEgSYsHmR3shSjf9ikqABRQQAcIBQAAAAAAACADqioBu9lanJS\/4e68gBgE1YEBAAABAQgKZPSVcMK4jAQXAwMAT0+KQ56NjlMHGW+d6G5ddduewRHnDyQJNOhFGSBeS16m4KVAja7XHlyuQrxKoq24Sn8bLVvUYgiRl0ogV926yAF+\/eBnK0DefdFCPgWpP6kXAwMAIh\/Eke2gVwnwKuWIWa9HbFAoJdRk5f1TigycRztSwvhmbFo="}
 00494{"packet_event_id":2,"packet_event_name":"packet-flow","thread_id":0,"packet_id":9663,"source":"pinterest.pcap","alias":"nDPId-test","flow_id":22,"flow_packet_id":3,"flow_last_seen":1605289716192,"flow_idle_time":7440000,"pkt_oversize":false,"pkt_caplen":86,"pkt_type":34525,"pkt_l3_offset":14,"pkt_l4_offset":54,"pkt_len":86,"pkt_l4_len":32,"ts_msec":1605289716192,"pkt":"5EKm5WPyqtsDr8lkht1gAAAAACAGPSoAFFBABwgFAAAAAAAAIAMqAcsBIEmLB5kd7IUo3\/YpAbuqKr\/h7rzZWpyUgBALf8h0AAABAQgKwrkp2GT0lXA="}
+00643{"flow_event_id":5,"flow_event_name":"guessed","thread_id":0,"packet_id":9768,"source":"pinterest.pcap","alias":"nDPId-test","flow_id":22,"flow_state":"info","flow_packets_processed":32,"flow_first_seen":1605289716168,"flow_last_seen":1605289716199,"flow_idle_time":7440000,"flow_min_l4_payload_len":0,"flow_max_l4_payload_len":1208,"flow_tot_l4_payload_len":21339,"flow_avg_l4_payload_len":666,"midstream":1,"ts_msec":1605289716199,"l3_proto":"ip6","src_ip":"2a01:cb01:2049:8b07:991d:ec85:28df:f629","dst_ip":"2a00:1450:4007:805::2003","src_port":43562,"dst_port":443,"l4_proto":"tcp","ndpi": {"proto":"TLS","breed":"Safe","category":"Web"}}
+00644{"flow_event_id":6,"flow_event_name":"detected","thread_id":0,"packet_id":9768,"source":"pinterest.pcap","alias":"nDPId-test","flow_id":22,"flow_state":"info","flow_packets_processed":32,"flow_first_seen":1605289716168,"flow_last_seen":1605289716199,"flow_idle_time":7440000,"flow_min_l4_payload_len":0,"flow_max_l4_payload_len":1208,"flow_tot_l4_payload_len":21339,"flow_avg_l4_payload_len":666,"midstream":1,"ts_msec":1605289716199,"l3_proto":"ip6","src_ip":"2a01:cb01:2049:8b07:991d:ec85:28df:f629","dst_ip":"2a00:1450:4007:805::2003","src_port":43562,"dst_port":443,"l4_proto":"tcp","ndpi": {"proto":"TLS","breed":"Safe","category":"Web"}}
 00613{"flow_event_id":1,"flow_event_name":"new","thread_id":0,"packet_id":14612,"source":"pinterest.pcap","alias":"nDPId-test","flow_id":23,"flow_state":"info","flow_packets_processed":1,"flow_first_seen":1605289717548,"flow_last_seen":1605289717548,"flow_idle_time":7440000,"flow_min_l4_payload_len":0,"flow_max_l4_payload_len":0,"flow_tot_l4_payload_len":0,"flow_avg_l4_payload_len":0,"midstream":0,"ts_msec":1605289717548,"l3_proto":"ip6","src_ip":"2a01:cb01:2049:8b07:991d:ec85:28df:f629","dst_ip":"2a00:1450:4007:816::200d","src_port":40894,"dst_port":443,"l4_proto":"tcp","flow_datalink":1,"flow_max_packets":3}
 00505{"packet_event_id":2,"packet_event_name":"packet-flow","thread_id":0,"packet_id":14612,"source":"pinterest.pcap","alias":"nDPId-test","flow_id":23,"flow_packet_id":1,"flow_last_seen":1605289717548,"flow_idle_time":7440000,"pkt_oversize":false,"pkt_caplen":94,"pkt_type":34525,"pkt_l3_offset":14,"pkt_l4_offset":54,"pkt_len":94,"pkt_l4_len":40,"ts_msec":1605289717548,"pkt":"qtsDr8lk5EKm5WPyht1gD67DACgGQCoBywEgSYsHmR3shSjf9ikqABRQQAcIFgAAAAAAACANn74Bu+7PaD4AAAAAoAL9ID+FAAACBAWgBAIICjGG9eUAAAAAAQMDBw=="}
 00507{"packet_event_id":2,"packet_event_name":"packet-flow","thread_id":0,"packet_id":14613,"source":"pinterest.pcap","alias":"nDPId-test","flow_id":23,"flow_packet_id":2,"flow_last_seen":1605289717572,"flow_idle_time":7440000,"pkt_oversize":false,"pkt_caplen":94,"pkt_type":34525,"pkt_l3_offset":14,"pkt_l4_offset":54,"pkt_len":94,"pkt_l4_len":40,"ts_msec":1605289717572,"pkt":"5EKm5WPyqtsDr8lkht1gAAAAACgGPSoAFFBABwgWAAAAAAAAIA0qAcsBIEmLB5kd7IUo3\/YpAbufvovR75juz2g\/oBJXgHfiAAACBAV4AQMDAwQCCArCuS86MYb15Q=="}
@@ -197,8 +199,7 @@
 00606{"flow_event_id":3,"flow_event_name":"idle","thread_id":0,"packet_id":17657,"source":"pinterest.pcap","alias":"nDPId-test","flow_id":28,"flow_state":"info","flow_packets_processed":2,"flow_first_seen":1605289720502,"flow_last_seen":1605289720592,"flow_idle_time":7440000,"flow_min_l4_payload_len":0,"flow_max_l4_payload_len":0,"flow_tot_l4_payload_len":0,"flow_avg_l4_payload_len":0,"midstream":1,"ts_msec":1605289734948,"l3_proto":"ip6","src_ip":"2a01:cb01:2049:8b07:991d:ec85:28df:f629","dst_ip":"2a04:4e42:1d::84","src_port":38402,"dst_port":443,"l4_proto":"tcp","flow_datalink":1,"flow_max_packets":3}
 00626{"flow_event_id":5,"flow_event_name":"guessed","thread_id":0,"packet_id":17657,"source":"pinterest.pcap","alias":"nDPId-test","flow_id":30,"flow_state":"info","flow_packets_processed":2,"flow_first_seen":1605289722610,"flow_last_seen":1605289722642,"flow_idle_time":7440000,"flow_min_l4_payload_len":0,"flow_max_l4_payload_len":0,"flow_tot_l4_payload_len":0,"flow_avg_l4_payload_len":0,"midstream":1,"ts_msec":1605289734948,"l3_proto":"ip6","src_ip":"2a01:cb01:2049:8b07:991d:ec85:28df:f629","dst_ip":"2a04:4e42:1d::84","src_port":38406,"dst_port":443,"l4_proto":"tcp","ndpi": {"proto":"TLS","breed":"Safe","category":"Web"}}
 00606{"flow_event_id":3,"flow_event_name":"idle","thread_id":0,"packet_id":17657,"source":"pinterest.pcap","alias":"nDPId-test","flow_id":30,"flow_state":"info","flow_packets_processed":2,"flow_first_seen":1605289722610,"flow_last_seen":1605289722642,"flow_idle_time":7440000,"flow_min_l4_payload_len":0,"flow_max_l4_payload_len":0,"flow_tot_l4_payload_len":0,"flow_avg_l4_payload_len":0,"midstream":1,"ts_msec":1605289734948,"l3_proto":"ip6","src_ip":"2a01:cb01:2049:8b07:991d:ec85:28df:f629","dst_ip":"2a04:4e42:1d::84","src_port":38406,"dst_port":443,"l4_proto":"tcp","flow_datalink":1,"flow_max_packets":3}
-00644{"flow_event_id":5,"flow_event_name":"guessed","thread_id":0,"packet_id":17657,"source":"pinterest.pcap","alias":"nDPId-test","flow_id":22,"flow_state":"info","flow_packets_processed":50,"flow_first_seen":1605289716168,"flow_last_seen":1605289716373,"flow_idle_time":7440000,"flow_min_l4_payload_len":0,"flow_max_l4_payload_len":1208,"flow_tot_l4_payload_len":32185,"flow_avg_l4_payload_len":643,"midstream":1,"ts_msec":1605289734948,"l3_proto":"ip6","src_ip":"2a01:cb01:2049:8b07:991d:ec85:28df:f629","dst_ip":"2a00:1450:4007:805::2003","src_port":43562,"dst_port":443,"l4_proto":"tcp","ndpi": {"proto":"TLS","breed":"Safe","category":"Web"}}
-00624{"flow_event_id":3,"flow_event_name":"idle","thread_id":0,"packet_id":17657,"source":"pinterest.pcap","alias":"nDPId-test","flow_id":22,"flow_state":"info","flow_packets_processed":50,"flow_first_seen":1605289716168,"flow_last_seen":1605289716373,"flow_idle_time":7440000,"flow_min_l4_payload_len":0,"flow_max_l4_payload_len":1208,"flow_tot_l4_payload_len":32185,"flow_avg_l4_payload_len":643,"midstream":1,"ts_msec":1605289734948,"l3_proto":"ip6","src_ip":"2a01:cb01:2049:8b07:991d:ec85:28df:f629","dst_ip":"2a00:1450:4007:805::2003","src_port":43562,"dst_port":443,"l4_proto":"tcp","flow_datalink":1,"flow_max_packets":3}
+00684{"flow_event_id":3,"flow_event_name":"idle","thread_id":0,"packet_id":17657,"source":"pinterest.pcap","alias":"nDPId-test","flow_id":22,"flow_state":"finished","flow_packets_processed":50,"flow_first_seen":1605289716168,"flow_last_seen":1605289716373,"flow_idle_time":7440000,"flow_min_l4_payload_len":0,"flow_max_l4_payload_len":1208,"flow_tot_l4_payload_len":32185,"flow_avg_l4_payload_len":643,"midstream":1,"ts_msec":1605289734948,"l3_proto":"ip6","src_ip":"2a01:cb01:2049:8b07:991d:ec85:28df:f629","dst_ip":"2a00:1450:4007:805::2003","src_port":43562,"dst_port":443,"l4_proto":"tcp","flow_datalink":1,"flow_max_packets":3,"ndpi": {"proto":"TLS","breed":"Safe","category":"Web"}}
 00634{"flow_event_id":5,"flow_event_name":"guessed","thread_id":0,"packet_id":17657,"source":"pinterest.pcap","alias":"nDPId-test","flow_id":32,"flow_state":"info","flow_packets_processed":2,"flow_first_seen":1605289726582,"flow_last_seen":1605289726637,"flow_idle_time":7440000,"flow_min_l4_payload_len":0,"flow_max_l4_payload_len":0,"flow_tot_l4_payload_len":0,"flow_avg_l4_payload_len":0,"midstream":1,"ts_msec":1605289734948,"l3_proto":"ip6","src_ip":"2a01:cb01:2049:8b07:991d:ec85:28df:f629","dst_ip":"2a00:1450:4007:816::200a","src_port":47682,"dst_port":443,"l4_proto":"tcp","ndpi": {"proto":"TLS","breed":"Safe","category":"Web"}}
 00614{"flow_event_id":3,"flow_event_name":"idle","thread_id":0,"packet_id":17657,"source":"pinterest.pcap","alias":"nDPId-test","flow_id":32,"flow_state":"info","flow_packets_processed":2,"flow_first_seen":1605289726582,"flow_last_seen":1605289726637,"flow_idle_time":7440000,"flow_min_l4_payload_len":0,"flow_max_l4_payload_len":0,"flow_tot_l4_payload_len":0,"flow_avg_l4_payload_len":0,"midstream":1,"ts_msec":1605289734948,"l3_proto":"ip6","src_ip":"2a01:cb01:2049:8b07:991d:ec85:28df:f629","dst_ip":"2a00:1450:4007:816::200a","src_port":47682,"dst_port":443,"l4_proto":"tcp","flow_datalink":1,"flow_max_packets":3}
 00709{"flow_event_id":3,"flow_event_name":"idle","thread_id":0,"packet_id":17657,"source":"pinterest.pcap","alias":"nDPId-test","flow_id":19,"flow_state":"finished","flow_packets_processed":71,"flow_first_seen":1605289715274,"flow_last_seen":1605289715612,"flow_idle_time":7440000,"flow_min_l4_payload_len":0,"flow_max_l4_payload_len":12420,"flow_tot_l4_payload_len":67017,"flow_avg_l4_payload_len":943,"midstream":0,"ts_msec":1605289734948,"l3_proto":"ip6","src_ip":"2a01:cb01:2049:8b07:991d:ec85:28df:f629","dst_ip":"2a03:2880:f030:13:face:b00c::3","src_port":51292,"dst_port":443,"l4_proto":"tcp","flow_datalink":1,"flow_max_packets":3,"ndpi": {"proto":"TLS.Facebook","breed":"Fun","category":"SocialNetwork"}}
@@ -236,12 +237,12 @@
 00614{"flow_event_id":3,"flow_event_name":"idle","thread_id":0,"packet_id":17657,"source":"pinterest.pcap","alias":"nDPId-test","flow_id":27,"flow_state":"info","flow_packets_processed":2,"flow_first_seen":1605289718347,"flow_last_seen":1605289718378,"flow_idle_time":7440000,"flow_min_l4_payload_len":0,"flow_max_l4_payload_len":0,"flow_tot_l4_payload_len":0,"flow_avg_l4_payload_len":0,"midstream":1,"ts_msec":1605289734948,"l3_proto":"ip6","src_ip":"2a01:cb01:2049:8b07:991d:ec85:28df:f629","dst_ip":"2a00:1450:4007:80c::200a","src_port":57130,"dst_port":443,"l4_proto":"tcp","flow_datalink":1,"flow_max_packets":3}
 00627{"flow_event_id":5,"flow_event_name":"guessed","thread_id":0,"packet_id":17657,"source":"pinterest.pcap","alias":"nDPId-test","flow_id":29,"flow_state":"info","flow_packets_processed":2,"flow_first_seen":1605289722442,"flow_last_seen":1605289722621,"flow_idle_time":7440000,"flow_min_l4_payload_len":0,"flow_max_l4_payload_len":0,"flow_tot_l4_payload_len":0,"flow_avg_l4_payload_len":0,"midstream":1,"ts_msec":1605289734948,"l3_proto":"ip6","src_ip":"2a01:cb01:2049:8b07:991d:ec85:28df:f629","dst_ip":"2600:1901::7a0b::","src_port":46918,"dst_port":443,"l4_proto":"tcp","ndpi": {"proto":"TLS","breed":"Safe","category":"Web"}}
 00607{"flow_event_id":3,"flow_event_name":"idle","thread_id":0,"packet_id":17657,"source":"pinterest.pcap","alias":"nDPId-test","flow_id":29,"flow_state":"info","flow_packets_processed":2,"flow_first_seen":1605289722442,"flow_last_seen":1605289722621,"flow_idle_time":7440000,"flow_min_l4_payload_len":0,"flow_max_l4_payload_len":0,"flow_tot_l4_payload_len":0,"flow_avg_l4_payload_len":0,"midstream":1,"ts_msec":1605289734948,"l3_proto":"ip6","src_ip":"2a01:cb01:2049:8b07:991d:ec85:28df:f629","dst_ip":"2600:1901::7a0b::","src_port":46918,"dst_port":443,"l4_proto":"tcp","flow_datalink":1,"flow_max_packets":3}
-00162{"daemon_event_id":3,"daemon_event_name":"shutdown","thread_id":0,"packet_id":17657,"source":"pinterest.pcap","alias":"nDPId-test","total-events-serialized":239}
+00162{"daemon_event_id":3,"daemon_event_name":"shutdown","thread_id":0,"packet_id":17657,"source":"pinterest.pcap","alias":"nDPId-test","total-events-serialized":240}
 ~~~~~~~~~~~~~~~~~~~~ SUMMARY ~~~~~~~~~~~~~~~~~~~~
 ~~ packets captured/processed: 17657/17657
 ~~ skipped flows.............: 0
 ~~ total layer4 data length..: 26490343 bytes
-~~ total detected protocols..: 20
+~~ total detected protocols..: 21
 ~~ total active/idle flows...: 37/37
 ~~ total timeout flows.......: 0
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -251,4 +252,4 @@
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 ~~ json string min len.......: 167 chars
 ~~ json string max len.......: 3191 chars
-~~ json string avg len.......: 1749 chars
+~~ json string avg len.......: 1748 chars