Major nDPId extension. Sorry for the huge commit.

 - nDPId: fixed invalid IP4/IP6 tuple compare
 - nDPIsrvd: fixed caching issue (finally)
 - added tiny c example (can be used to check flow manager sanity)
 - c-captured: use flow_last_seen timestamp from `struct nDPIsrvd_flow`
 - README.md update: added example JSON sequence
 - nDPId: added new flow event `update` necessary for correct
   timeout handling (and other future use-cases)
 - nDPIsrvd.h and nDPIsrvd.py: switched to an instance
   (consists of an alias/source tuple) based flow manager
 - every flow related event **must** now serialize `alias`, `source`,
   `flow_id`, `flow_last_seen` and `flow_idle_time` to make the timeout
   handling and verification process work correctly
 - nDPIsrvd.h: ability to profile any dynamic memory (de-)allocation
 - nDPIsrvd.py: removed PcapPacket class (unused)
 - py-flow-dashboard and py-flow-multiprocess: fixed race condition
 - py-flow-info: print statusbar with probably useful information
 - nDPId/nDPIsrvd.h: switched from packet-flow only timestamps (`pkt_*sec`)
   to a generic flow event timestamp `ts_msec`
 - nDPId-test: added additional checks
 - nDPId: increased ICMP flow timeout
 - nDPId: using event based i/o if capturing packets from a device
 - nDPIsrvd: fixed memory leak on shutdown if remote descriptors
   were still connected

Signed-off-by: Toni Uhlig <matzeton@googlemail.com>

1 files changed, 15 insertions, 22 deletions

diff --git a/test/results/tls_alert.pcap.out b/test/results/tls_alert.pcap.out
index c20d28c85..3f9c5b471 100644
--- a/test/results/tls_alert.pcap.out
+++ b/test/results/tls_alert.pcap.out
@@ -1,30 +1,23 @@
-00477{"daemon_event_id":1,"daemon_event_name":"init","thread_id":0,"packet_id":0,"source":"tls_alert.pcap","alias":"nDPId-test","max-flows-per-thread":2048,"max-idle-flows-per-thread":256,"tick-resolution":1000,"reader-thread-count":1,"idle-scan-period":10000,"generic-max-idle-time":600000,"icmp-max-idle-time":10000,"udp-max-idle-time":180000,"tcp-max-idle-time":7440000,"tcp-max-post-end-flow-time":120000,"max-packets-per-flow-to-send":15,"max-packets-per-flow-to-process":255}
-00494{"flow_event_id":1,"flow_event_name":"new","thread_id":0,"packet_id":1,"source":"tls_alert.pcap","alias":"nDPId-test","flow_id":1,"flow_packet_id":1,"flow_first_seen":1628259176203,"flow_last_seen":1628259176203,"flow_min_l4_payload_len":0,"flow_max_l4_payload_len":0,"flow_tot_l4_payload_len":0,"flow_avg_l4_payload_len":0,"midstream":0,"l3_proto":"ip4","src_ip":"192.168.1.192","dst_ip":"192.168.1.20","src_port":63158,"dst_port":443,"l4_proto":"tcp","flow_datalink":1,"flow_max_packets":15}
-00442{"flow_id":1,"flow_packet_id":1,"packet_event_id":2,"packet_event_name":"packet-flow","thread_id":0,"packet_id":1,"source":"tls_alert.pcap","alias":"nDPId-test","pkt_oversize":false,"pkt_ts_sec":1628259176,"pkt_ts_usec":203392,"pkt_caplen":78,"pkt_type":2048,"pkt_l3_offset":14,"pkt_l4_offset":34,"pkt_len":78,"pkt_l4_len":44,"pkt":"AICPmq69oM7IELEuCABFAABAAABAAEAGtpPAqAHAwKgBFPa2AbvtIEkOAAAAALAC\/\/9MagAAAgQFtAEDAwUBAQgKE9Ij+wAAAAAEAgAA"}
-00436{"flow_id":1,"flow_packet_id":2,"packet_event_id":2,"packet_event_name":"packet-flow","thread_id":0,"packet_id":2,"source":"tls_alert.pcap","alias":"nDPId-test","pkt_oversize":false,"pkt_ts_sec":1628259176,"pkt_ts_usec":203813,"pkt_caplen":74,"pkt_type":2048,"pkt_l3_offset":14,"pkt_l4_offset":34,"pkt_len":74,"pkt_l4_len":40,"pkt":"oM7IELEuAICPmq69CABFAAA8AABAAEAGtpfAqAEUwKgBwAG79rbEoc1F7SBJD6AScSBz9QAAAgQFtAQCCAoAseWtE9Ij+wEDAwc="}
-00426{"flow_id":1,"flow_packet_id":3,"packet_event_id":2,"packet_event_name":"packet-flow","thread_id":0,"packet_id":3,"source":"tls_alert.pcap","alias":"nDPId-test","pkt_oversize":false,"pkt_ts_sec":1628259176,"pkt_ts_usec":203877,"pkt_caplen":66,"pkt_type":2048,"pkt_l3_offset":14,"pkt_l4_offset":34,"pkt_len":66,"pkt_l4_len":32,"pkt":"AICPmq69oM7IELEuCABFAAA0AABAAEAGtp\/AqAHAwKgBFPa2AbvtIEkPxKHNRoAQEBUDzQAAAQEIChPSI\/sAseWt"}
-00700{"flow_id":1,"flow_packet_id":4,"packet_event_id":2,"packet_event_name":"packet-flow","thread_id":0,"packet_id":4,"source":"tls_alert.pcap","alias":"nDPId-test","pkt_oversize":false,"pkt_ts_sec":1628259176,"pkt_ts_usec":204397,"pkt_caplen":265,"pkt_type":2048,"pkt_l3_offset":14,"pkt_l4_offset":34,"pkt_len":265,"pkt_l4_len":231,"pkt":"AICPmq69oM7IELEuCABFAAD7AABAAEAGtdjAqAHAwKgBFPa2AbvtIEkPxKHNRoAYEBXUyQAAAQEIChPSI\/wAseWtFgMBAMIBAAC+AwFS2zXz6qEYi\/Hhk\/zPMz5Yc\/Q1u9wcSBgXhT9UdiVqOgAAEgAvADMANQA5wAnACsATwBRWAAEAAIP\/AQABAAAAAB0AGwAAGHd3dy5nb29nbGUtYW5hbHl0aWNzLmNvbQAXAAAABQAFAQAAAAAzdAAAABIAAAAQADAALgJoMgVoMi0xNgVoMi0xNQVoMi0xNAhzcGR5LzMuMQZzcGR5LzMIaHR0cC8xLjEACwACAQAACgAKAAgAHQAXABgAGQ=="}
-00841{"flow_event_id":5,"flow_event_name":"detected","thread_id":0,"packet_id":4,"source":"tls_alert.pcap","alias":"nDPId-test","flow_id":1,"flow_packet_id":4,"flow_first_seen":1628259176203,"flow_last_seen":1628259176204,"flow_min_l4_payload_len":0,"flow_max_l4_payload_len":199,"flow_tot_l4_payload_len":199,"flow_avg_l4_payload_len":49,"midstream":0,"l3_proto":"ip4","src_ip":"192.168.1.192","dst_ip":"192.168.1.20","src_port":63158,"dst_port":443,"l4_proto":"tcp","ndpi": {"flow_risk": {"7":"Obsolete TLS version (older than 1.2)"},"proto":"TLS.Google","breed":"Tracker\/Ads","category":"Web"},"tls": {"version":"TLSv1","client_requested_server_name":"www.google-analytics.com","ja3":"d78489b860c8bf7838a6ff0b4d131541","ja3s":"","unsafe_cipher":0,"cipher":"TLS_NULL_WITH_NULL_NULL","alpn":"h2,h2-16,h2-15,h2-14,spdy\/3.1,spdy\/3,http\/1.1"}}
-00424{"flow_id":1,"flow_packet_id":5,"packet_event_id":2,"packet_event_name":"packet-flow","thread_id":0,"packet_id":5,"source":"tls_alert.pcap","alias":"nDPId-test","pkt_oversize":false,"pkt_ts_sec":1628259176,"pkt_ts_usec":204809,"pkt_caplen":66,"pkt_type":2048,"pkt_l3_offset":14,"pkt_l4_offset":34,"pkt_len":66,"pkt_l4_len":32,"pkt":"oM7IELEuAICPmq69CABFAAA0KOtAAEAGjbTAqAEUwKgBwAG79rbEoc1G7SBJ1oAQAOsSLwAAAQEICgCx5a0T0iP8"}
-00436{"flow_id":1,"flow_packet_id":6,"packet_event_id":2,"packet_event_name":"packet-flow","thread_id":0,"packet_id":6,"source":"tls_alert.pcap","alias":"nDPId-test","pkt_oversize":false,"pkt_ts_sec":1628259176,"pkt_ts_usec":204934,"pkt_caplen":73,"pkt_type":2048,"pkt_l3_offset":14,"pkt_l4_offset":34,"pkt_len":73,"pkt_l4_len":39,"pkt":"oM7IELEuAICPmq69CABFAAA7KOxAAEAGjazAqAEUwKgBwAG79rbEoc1G7SBJ1oAYAOu0GgAAAQEICgCx5a0T0iP8FQMBAAICRg=="}
-00427{"flow_id":1,"flow_packet_id":7,"packet_event_id":2,"packet_event_name":"packet-flow","thread_id":0,"packet_id":7,"source":"tls_alert.pcap","alias":"nDPId-test","pkt_oversize":false,"pkt_ts_sec":1628259176,"pkt_ts_usec":204981,"pkt_caplen":66,"pkt_type":2048,"pkt_l3_offset":14,"pkt_l4_offset":34,"pkt_len":66,"pkt_l4_len":32,"pkt":"AICPmq69oM7IELEuCABFAAA0AABAAEAGtp\/AqAHAwKgBFPa2AbvtIEnWxKHNTYAQEBUC\/gAAAQEIChPSI\/wAseWt"}
-00424{"flow_id":1,"flow_packet_id":8,"packet_event_id":2,"packet_event_name":"packet-flow","thread_id":0,"packet_id":8,"source":"tls_alert.pcap","alias":"nDPId-test","pkt_oversize":false,"pkt_ts_sec":1628259176,"pkt_ts_usec":205061,"pkt_caplen":66,"pkt_type":2048,"pkt_l3_offset":14,"pkt_l4_offset":34,"pkt_len":66,"pkt_l4_len":32,"pkt":"oM7IELEuAICPmq69CABFAAA0KO1AAEAGjbLAqAEUwKgBwAG79rbEoc1N7SBJ1oARAOsSJwAAAQEICgCx5a0T0iP8"}
-00427{"flow_id":1,"flow_packet_id":9,"packet_event_id":2,"packet_event_name":"packet-flow","thread_id":0,"packet_id":9,"source":"tls_alert.pcap","alias":"nDPId-test","pkt_oversize":false,"pkt_ts_sec":1628259176,"pkt_ts_usec":205093,"pkt_caplen":66,"pkt_type":2048,"pkt_l3_offset":14,"pkt_l4_offset":34,"pkt_len":66,"pkt_l4_len":32,"pkt":"AICPmq69oM7IELEuCABFAAA0AABAAEAGtp\/AqAHAwKgBFPa2AbvtIEnWxKHNToAQEBUC\/QAAAQEIChPSI\/wAseWt"}
-00428{"flow_id":1,"flow_packet_id":10,"packet_event_id":2,"packet_event_name":"packet-flow","thread_id":0,"packet_id":10,"source":"tls_alert.pcap","alias":"nDPId-test","pkt_oversize":false,"pkt_ts_sec":1628259176,"pkt_ts_usec":205826,"pkt_caplen":66,"pkt_type":2048,"pkt_l3_offset":14,"pkt_l4_offset":34,"pkt_len":66,"pkt_l4_len":32,"pkt":"AICPmq69oM7IELEuCABFAAA0AABAAEAGtp\/AqAHAwKgBFPa2AbvtIEnWxKHNToAREBUC+wAAAQEIChPSI\/0AseWt"}
-00426{"flow_id":1,"flow_packet_id":11,"packet_event_id":2,"packet_event_name":"packet-flow","thread_id":0,"packet_id":11,"source":"tls_alert.pcap","alias":"nDPId-test","pkt_oversize":false,"pkt_ts_sec":1628259176,"pkt_ts_usec":206182,"pkt_caplen":66,"pkt_type":2048,"pkt_l3_offset":14,"pkt_l4_offset":34,"pkt_len":66,"pkt_l4_len":32,"pkt":"oM7IELEuAICPmq69CABFAAA0S9xAAEAGasPAqAEUwKgBwAG79rbEoc1O7SBJ14AQAOsSJAAAAQEICgCx5a4T0iP9"}
-00501{"flow_event_id":2,"flow_event_name":"end","thread_id":0,"packet_id":11,"source":"tls_alert.pcap","alias":"nDPId-test","flow_id":1,"flow_packet_id":11,"flow_first_seen":1628259176203,"flow_last_seen":1628259176206,"flow_min_l4_payload_len":0,"flow_max_l4_payload_len":199,"flow_tot_l4_payload_len":206,"flow_avg_l4_payload_len":18,"midstream":0,"l3_proto":"ip4","src_ip":"192.168.1.192","dst_ip":"192.168.1.20","src_port":63158,"dst_port":443,"l4_proto":"tcp","flow_datalink":1,"flow_max_packets":15}
-00129{"daemon_event_id":3,"daemon_event_name":"shutdown","thread_id":0,"packet_id":11,"source":"tls_alert.pcap","alias":"nDPId-test"}
+00443{"daemon_event_id":1,"daemon_event_name":"init","thread_id":0,"packet_id":0,"source":"tls_alert.pcap","alias":"nDPId-test","max-flows-per-thread":2048,"max-idle-flows-per-thread":256,"tick-resolution":1000,"reader-thread-count":1,"flow-scan-interval":10000,"generic-max-idle-time":600000,"icmp-max-idle-time":120000,"udp-max-idle-time":180000,"tcp-max-idle-time":7460000,"max-packets-per-flow-to-send":3,"max-packets-per-flow-to-process":255}
+00550{"flow_event_id":1,"flow_event_name":"new","thread_id":0,"packet_id":1,"source":"tls_alert.pcap","alias":"nDPId-test","flow_id":1,"flow_packets_processed":1,"flow_first_seen":1628259176203,"flow_last_seen":1628259176203,"flow_idle_time":7440000,"flow_min_l4_payload_len":0,"flow_max_l4_payload_len":0,"flow_tot_l4_payload_len":0,"flow_avg_l4_payload_len":0,"midstream":0,"ts_msec":1628259176203,"l3_proto":"ip4","src_ip":"192.168.1.192","dst_ip":"192.168.1.20","src_port":63158,"dst_port":443,"l4_proto":"tcp","flow_datalink":1,"flow_max_packets":3}
+00477{"packet_event_id":2,"packet_event_name":"packet-flow","thread_id":0,"packet_id":1,"source":"tls_alert.pcap","alias":"nDPId-test","flow_id":1,"flow_packet_id":1,"flow_last_seen":1628259176203,"flow_idle_time":7440000,"pkt_oversize":false,"pkt_caplen":78,"pkt_type":2048,"pkt_l3_offset":14,"pkt_l4_offset":34,"pkt_len":78,"pkt_l4_len":44,"ts_msec":1628259176203,"pkt":"AICPmq69oM7IELEuCABFAABAAABAAEAGtpPAqAHAwKgBFPa2AbvtIEkOAAAAALAC\/\/9MagAAAgQFtAEDAwUBAQgKE9Ij+wAAAAAEAgAA"}
+00471{"packet_event_id":2,"packet_event_name":"packet-flow","thread_id":0,"packet_id":2,"source":"tls_alert.pcap","alias":"nDPId-test","flow_id":1,"flow_packet_id":2,"flow_last_seen":1628259176203,"flow_idle_time":7440000,"pkt_oversize":false,"pkt_caplen":74,"pkt_type":2048,"pkt_l3_offset":14,"pkt_l4_offset":34,"pkt_len":74,"pkt_l4_len":40,"ts_msec":1628259176203,"pkt":"oM7IELEuAICPmq69CABFAAA8AABAAEAGtpfAqAEUwKgBwAG79rbEoc1F7SBJD6AScSBz9QAAAgQFtAQCCAoAseWtE9Ij+wEDAwc="}
+00461{"packet_event_id":2,"packet_event_name":"packet-flow","thread_id":0,"packet_id":3,"source":"tls_alert.pcap","alias":"nDPId-test","flow_id":1,"flow_packet_id":3,"flow_last_seen":1628259176203,"flow_idle_time":7440000,"pkt_oversize":false,"pkt_caplen":66,"pkt_type":2048,"pkt_l3_offset":14,"pkt_l4_offset":34,"pkt_len":66,"pkt_l4_len":32,"ts_msec":1628259176203,"pkt":"AICPmq69oM7IELEuCABFAAA0AABAAEAGtp\/AqAHAwKgBFPa2AbvtIEkPxKHNRoAQEBUDzQAAAQEIChPSI\/sAseWt"}
+00898{"flow_event_id":6,"flow_event_name":"detected","thread_id":0,"packet_id":4,"source":"tls_alert.pcap","alias":"nDPId-test","flow_id":1,"flow_packets_processed":4,"flow_first_seen":1628259176203,"flow_last_seen":1628259176204,"flow_idle_time":7440000,"flow_min_l4_payload_len":0,"flow_max_l4_payload_len":199,"flow_tot_l4_payload_len":199,"flow_avg_l4_payload_len":49,"midstream":0,"ts_msec":1628259176204,"l3_proto":"ip4","src_ip":"192.168.1.192","dst_ip":"192.168.1.20","src_port":63158,"dst_port":443,"l4_proto":"tcp","ndpi": {"flow_risk": {"7":"Obsolete TLS version (older than 1.2)"},"proto":"TLS.Google","breed":"Tracker\/Ads","category":"Web"},"tls": {"version":"TLSv1","client_requested_server_name":"www.google-analytics.com","ja3":"d78489b860c8bf7838a6ff0b4d131541","ja3s":"","unsafe_cipher":0,"cipher":"TLS_NULL_WITH_NULL_NULL","alpn":"h2,h2-16,h2-15,h2-14,spdy\/3.1,spdy\/3,http\/1.1"}}
+00557{"flow_event_id":2,"flow_event_name":"end","thread_id":0,"packet_id":11,"source":"tls_alert.pcap","alias":"nDPId-test","flow_id":1,"flow_packets_processed":11,"flow_first_seen":1628259176203,"flow_last_seen":1628259176206,"flow_idle_time":7440000,"flow_min_l4_payload_len":0,"flow_max_l4_payload_len":199,"flow_tot_l4_payload_len":206,"flow_avg_l4_payload_len":18,"midstream":0,"ts_msec":1628259176206,"l3_proto":"ip4","src_ip":"192.168.1.192","dst_ip":"192.168.1.20","src_port":63158,"dst_port":443,"l4_proto":"tcp","flow_datalink":1,"flow_max_packets":3}
+00157{"daemon_event_id":3,"daemon_event_name":"shutdown","thread_id":0,"packet_id":11,"source":"tls_alert.pcap","alias":"nDPId-test","total-events-serialized":8}
 ~~~~~~~~~~~~~~~~~~~~ SUMMARY ~~~~~~~~~~~~~~~~~~~~
 ~~ packets captured/processed: 11/11
 ~~ skipped flows.............: 0
-~~ total layer4 data length..: 578 bytes
+~~ total layer4 data length..: 206 bytes
 ~~ total detected protocols..: 1
 ~~ total active/idle flows...: 1/1
+~~ total timeout flows.......: 0
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-~~ total memory allocated....: 1930481 bytes
-~~ total memory freed........: 1930481 bytes
+~~ total memory allocated....: 1930537 bytes
+~~ total memory freed........: 1930537 bytes
 ~~ total allocations/frees...: 35351/35351
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-~~ json string min len.......: 134 chars
-~~ json string max len.......: 846 chars
-~~ json string avg len.......: 563 chars
+~~ json string min len.......: 162 chars
+~~ json string max len.......: 903 chars
+~~ json string avg len.......: 579 chars