Major nDPId extension. Sorry for the huge commit.

 - nDPId: fixed invalid IP4/IP6 tuple compare
 - nDPIsrvd: fixed caching issue (finally)
 - added tiny c example (can be used to check flow manager sanity)
 - c-captured: use flow_last_seen timestamp from `struct nDPIsrvd_flow`
 - README.md update: added example JSON sequence
 - nDPId: added new flow event `update` necessary for correct
   timeout handling (and other future use-cases)
 - nDPIsrvd.h and nDPIsrvd.py: switched to an instance
   (consists of an alias/source tuple) based flow manager
 - every flow related event **must** now serialize `alias`, `source`,
   `flow_id`, `flow_last_seen` and `flow_idle_time` to make the timeout
   handling and verification process work correctly
 - nDPIsrvd.h: ability to profile any dynamic memory (de-)allocation
 - nDPIsrvd.py: removed PcapPacket class (unused)
 - py-flow-dashboard and py-flow-multiprocess: fixed race condition
 - py-flow-info: print statusbar with probably useful information
 - nDPId/nDPIsrvd.h: switched from packet-flow only timestamps (`pkt_*sec`)
   to a generic flow event timestamp `ts_msec`
 - nDPId-test: added additional checks
 - nDPId: increased ICMP flow timeout
 - nDPId: using event based i/o if capturing packets from a device
 - nDPIsrvd: fixed memory leak on shutdown if remote descriptors
   were still connected

Signed-off-by: Toni Uhlig <matzeton@googlemail.com>

1 files changed, 21 insertions, 20 deletions

diff --git a/test/results/encrypted_sni.pcap.out b/test/results/encrypted_sni.pcap.out
index 6f62f6495..89e6372bd 100644
--- a/test/results/encrypted_sni.pcap.out
+++ b/test/results/encrypted_sni.pcap.out
@@ -1,28 +1,29 @@
-00481{"daemon_event_id":1,"daemon_event_name":"init","thread_id":0,"packet_id":0,"source":"encrypted_sni.pcap","alias":"nDPId-test","max-flows-per-thread":2048,"max-idle-flows-per-thread":256,"tick-resolution":1000,"reader-thread-count":1,"idle-scan-period":10000,"generic-max-idle-time":600000,"icmp-max-idle-time":10000,"udp-max-idle-time":180000,"tcp-max-idle-time":7440000,"tcp-max-post-end-flow-time":120000,"max-packets-per-flow-to-send":15,"max-packets-per-flow-to-process":255}
-00506{"flow_event_id":1,"flow_event_name":"new","thread_id":0,"packet_id":1,"source":"encrypted_sni.pcap","alias":"nDPId-test","flow_id":1,"flow_packet_id":1,"flow_first_seen":1590680386576,"flow_last_seen":1590680386576,"flow_min_l4_payload_len":716,"flow_max_l4_payload_len":716,"flow_tot_l4_payload_len":716,"flow_avg_l4_payload_len":716,"midstream":1,"l3_proto":"ip4","src_ip":"192.168.1.12","dst_ip":"104.27.129.77","src_port":49886,"dst_port":443,"l4_proto":"tcp","flow_datalink":1,"flow_max_packets":15}
-01380{"flow_id":1,"flow_packet_id":1,"packet_event_id":2,"packet_event_name":"packet-flow","thread_id":0,"packet_id":1,"source":"encrypted_sni.pcap","alias":"nDPId-test","pkt_oversize":false,"pkt_ts_sec":1590680386,"pkt_ts_usec":576239,"pkt_caplen":770,"pkt_type":2048,"pkt_l3_offset":14,"pkt_l4_offset":34,"pkt_len":770,"pkt_l4_len":736,"pkt":"EBMx8Tl2KDc3AG3ICABFAAL0AABAAEAGjOfAqAEMaBuBTcLeAbt3Q5LX\/48DFVAYIACwHgAAFgMBAscBAALDAwOTwM86TEdZaYZx77QiKeLaOUyI6FPS+J3L+0S3MA31OCDtrXy2AkmiC5EC8aXH8NKs5TG5ofTGvlsmIWUcTFlOhgAkEwETAxMCwCvAL8ypzKjALMAwwArACcATwBQAMwA5AC8ANQAKAQACVgAXAAD\/AQABAAAKAA4ADAAdABcAGAAZAQABAQALAAIBAAAjAAAAEAAOAAwCaDIIaHR0cC8xLjEABQAFAQAAAAAAMwBrAGkAHQAg9C+VXLX0pUAYcvwRMlm2BfjMFL+A2Ha+teHeYm8XszAAFwBBBKhP+5j\/iIqKULsVEv1xkLdgIoxwczB5EVKfTq\/0aLaIOqqUx255GoGIKzaHGdYeWvgG2FTscntynOjMKiH+1xMAKwAJCAMEAwMDAgMBAA0AGAAWBAMFAwYDCAQIBQgGBAEFAQYBAgMCAQAtAAIBAf\/OAW4TAQAdACAoJey8d6KdccaSJO2lCYt20kw0EEYFyldVNE\/b+wVlLQAgHyQSymUyoBaYNvGbjOJlOzPcW4r7yiRdTxErCb+vUsgBJJYkyzxOIwgn94z1v2QNIt6jP8xZjqajLZOZBVhvvpl7nmhmH4lW1IkwcuGd4kzR+4ip9x\/EzAG6tckU\/flqZH1nG16JhZuu6rEiIYaISW303wwyjD1flAsQnOsqJ0PVy+NZQoiiKbjH4viDA+P+GiaonlAB8r2TaJD+948G4F7MBjpovbjBjfrBFM8f7NuL4fwv7ssjFdJ5mNaCsSn9Hj6115hdy9xFKhCCzMA44L9pVw\/vrGvG+5UfibZ5LK2nZAPALOtdzhzm7d0W1ff7a4XSuSSFRI3gCI5CHoPx4osmf747Wa4ElvuEUhPCcdTFrF6efl9qMHJEUwf8zrcwZxBFmZHEDMTcH8MlFUx5dN14A3E5eAVFahmuI+6IR1wd8HaXtmYAHAACQAE="}
-00790{"flow_event_id":5,"flow_event_name":"detected","thread_id":0,"packet_id":1,"source":"encrypted_sni.pcap","alias":"nDPId-test","flow_id":1,"flow_packet_id":1,"flow_first_seen":1590680386576,"flow_last_seen":1590680386576,"flow_min_l4_payload_len":716,"flow_max_l4_payload_len":716,"flow_tot_l4_payload_len":716,"flow_avg_l4_payload_len":716,"midstream":1,"l3_proto":"ip4","src_ip":"192.168.1.12","dst_ip":"104.27.129.77","src_port":49886,"dst_port":443,"l4_proto":"tcp","ndpi": {"proto":"TLS.Cloudflare","breed":"Acceptable","category":"Web"},"tls": {"version":"TLSv1.2","client_requested_server_name":"","ja3":"e5ef852e686954ba9fe060fbfa881e15","ja3s":"","unsafe_cipher":0,"cipher":"TLS_NULL_WITH_NULL_NULL","alpn":"h2,http\/1.1","tls_supported_versions":"TLSv1.3,TLSv1.2,TLSv1.1,TLSv1"}}
-00507{"flow_event_id":1,"flow_event_name":"new","thread_id":0,"packet_id":2,"source":"encrypted_sni.pcap","alias":"nDPId-test","flow_id":2,"flow_packet_id":1,"flow_first_seen":1590680387847,"flow_last_seen":1590680387847,"flow_min_l4_payload_len":716,"flow_max_l4_payload_len":716,"flow_tot_l4_payload_len":716,"flow_avg_l4_payload_len":716,"midstream":1,"l3_proto":"ip4","src_ip":"192.168.1.12","dst_ip":"104.16.125.175","src_port":49887,"dst_port":443,"l4_proto":"tcp","flow_datalink":1,"flow_max_packets":15}
-01384{"flow_id":2,"flow_packet_id":1,"packet_event_id":2,"packet_event_name":"packet-flow","thread_id":0,"packet_id":2,"source":"encrypted_sni.pcap","alias":"nDPId-test","pkt_oversize":false,"pkt_ts_sec":1590680387,"pkt_ts_usec":847337,"pkt_caplen":770,"pkt_type":2048,"pkt_l3_offset":14,"pkt_l4_offset":34,"pkt_len":770,"pkt_l4_len":736,"pkt":"EBMx8Tl2KDc3AG3ICABFAAL0AABAAEAGkJDAqAEMaBB9r8LfAbu98X4VZuCG7lAYIACqfgAAFgMBAscBAALDAwPZvt6xqK7JiSO2eRBioUk2Uu867QdPWpn6Sv4hYS472iAz8c+AKNafKEsBeorsjdYMXk2HdHvKJL23Af8gga\/qxAAkEwETAxMCwCvAL8ypzKjALMAwwArACcATwBQAMwA5AC8ANQAKAQACVgAXAAD\/AQABAAAKAA4ADAAdABcAGAAZAQABAQALAAIBAAAjAAAAEAAOAAwCaDIIaHR0cC8xLjEABQAFAQAAAAAAMwBrAGkAHQAg0HCVKAanlLS9J1B8hdchDfkoKDxcPc3B5hBZYsZWdz8AFwBBBCakAur\/e3rF+tGl0au7NOTY4DQpBg\/YjV6ew74w8otvaCGiCdoeWGhEGjsldqwZrBxN3o59i8BSdRX+YPQ+GgkAKwAJCAMEAwMDAgMBAA0AGAAWBAMFAwYDCAQIBQgGBAEFAQYBAgMCAQAtAAIBAf\/OAW4TAQAdACAFyK2kXV21yqtAW2T62b\/NDTnJgxOrhECle3qcjynhZQAgHyQSymUyoBaYNvGbjOJlOzPcW4r7yiRdTxErCb+vUsgBJLkAAE456EuY9a6HsKAg7En+2G8rSItqsoven5V2IfJ3Q2bekOZcTKgIZokRYkaF7ExtxsFhqXy+gigbwIQnaXqjvmpA5fAKz4tj4ykxew5OhWQtUKuHkOYZfaYtn1syOdzFlDd5f+dopSDJ1HH+q6E3XfYeSjmwk2PLEJ57JKeThEiW3dFrbufb5XbXZxYdeC179v7EU6Bakj2Njpvv\/Jfo5WxPGqtw\/pm8l4GeHZCKXzswlPS\/Jet6JKlP28PhB6QjuLs0HyKQD3u9h3gOMLbs85P+uPv\/61THn6BnP+Gq0XsiHUv\/ZFCqDNSvUTBmtmCAtgIUfzrLcUWkNsVonaILrLi\/m6vYUQElVuyPe7nXS\/qvJdz0NipXdWB8POXCwp8YOWkAHAACQAE="}
-00791{"flow_event_id":5,"flow_event_name":"detected","thread_id":0,"packet_id":2,"source":"encrypted_sni.pcap","alias":"nDPId-test","flow_id":2,"flow_packet_id":1,"flow_first_seen":1590680387847,"flow_last_seen":1590680387847,"flow_min_l4_payload_len":716,"flow_max_l4_payload_len":716,"flow_tot_l4_payload_len":716,"flow_avg_l4_payload_len":716,"midstream":1,"l3_proto":"ip4","src_ip":"192.168.1.12","dst_ip":"104.16.125.175","src_port":49887,"dst_port":443,"l4_proto":"tcp","ndpi": {"proto":"TLS.Cloudflare","breed":"Acceptable","category":"Web"},"tls": {"version":"TLSv1.2","client_requested_server_name":"","ja3":"e5ef852e686954ba9fe060fbfa881e15","ja3s":"","unsafe_cipher":0,"cipher":"TLS_NULL_WITH_NULL_NULL","alpn":"h2,http\/1.1","tls_supported_versions":"TLSv1.3,TLSv1.2,TLSv1.1,TLSv1"}}
-00506{"flow_event_id":1,"flow_event_name":"new","thread_id":0,"packet_id":3,"source":"encrypted_sni.pcap","alias":"nDPId-test","flow_id":3,"flow_packet_id":1,"flow_first_seen":1590680391590,"flow_last_seen":1590680391590,"flow_min_l4_payload_len":716,"flow_max_l4_payload_len":716,"flow_tot_l4_payload_len":716,"flow_avg_l4_payload_len":716,"midstream":1,"l3_proto":"ip4","src_ip":"192.168.1.12","dst_ip":"104.22.71.197","src_port":49897,"dst_port":443,"l4_proto":"tcp","flow_datalink":1,"flow_max_packets":15}
-01378{"flow_id":3,"flow_packet_id":1,"packet_event_id":2,"packet_event_name":"packet-flow","thread_id":0,"packet_id":3,"source":"encrypted_sni.pcap","alias":"nDPId-test","pkt_oversize":false,"pkt_ts_sec":1590680391,"pkt_ts_usec":590254,"pkt_caplen":770,"pkt_type":2048,"pkt_l3_offset":14,"pkt_l4_offset":34,"pkt_len":770,"pkt_l4_len":736,"pkt":"EBMx8Tl2KDc3AG3ICABFAAL0AABAAEAGxnTAqAEMaBZHxcLpAbsLJg40SW6gUlAYIAANXgAAFgMBAscBAALDAwMJLl9l\/OldUJYbpqd0xOpts3Kv4zg2hroTXcdX9KeB2CBjkfBVUTqX532YPuVZHQd0J5lIK2OZH9nsSRBnWwKDWwAkEwETAxMCwCvAL8ypzKjALMAwwArACcATwBQAMwA5AC8ANQAKAQACVgAXAAD\/AQABAAAKAA4ADAAdABcAGAAZAQABAQALAAIBAAAjAAAAEAAOAAwCaDIIaHR0cC8xLjEABQAFAQAAAAAAMwBrAGkAHQAgsbxhJX9IcnjB7rdgEb2YIBohnnxEhKIToNk1er8CIioAFwBBBLtlLNXLCuP0okhISXwuyj6tgeyLGZ5yaSZ9uT3zAbum2y5l1gYjS6RGBBL9dNcuY2pA4Ze582sOuuo0cAvw2TsAKwAJCAMEAwMDAgMBAA0AGAAWBAMFAwYDCAQIBQgGBAEFAQYBAgMCAQAtAAIBAf\/OAW4TAQAdACCgcq\/jSZGFwhXJHl9nfU84W9RHblecX+XHXi+knd++egAgHyQSymUyoBaYNvGbjOJlOzPcW4r7yiRdTxErCb+vUsgBJM1prHJ\/+qDqcKEqpG5xU365kjS5loGMkTxyoKwRhL+l3TthfgE+TKCSsunPt4vNjTPLrxKpdN+3jkm4v5pXmXQY7xTIeDCWHjyEgNKkvyfWHZEc70MAkkqfNhBXSLrthF\/1heQEBlRbs1xtqteJZDPsTf1rb0lyjahdcH23rHhPVaZljcat4wh7Hka7vt+kTz6HVLMaa8+FGdKR02KYBfqCbkN5nqbjMCHPCoPKBXF7APN9aYQZNPW1vyVMZGeIilksOKMAfbO31cu423QrZX+PlzwFC6qBeqVxOTzYpLwLIxJGCnfdBRD0u85D1TvPM05OjHVwJVu9F3FEA\/S2klQ0zWf5b6ngXXAHdoEO61eGscgYik1z+CCLYUuTKEqAk5KVlL4AHAACQAE="}
-00790{"flow_event_id":5,"flow_event_name":"detected","thread_id":0,"packet_id":3,"source":"encrypted_sni.pcap","alias":"nDPId-test","flow_id":3,"flow_packet_id":1,"flow_first_seen":1590680391590,"flow_last_seen":1590680391590,"flow_min_l4_payload_len":716,"flow_max_l4_payload_len":716,"flow_tot_l4_payload_len":716,"flow_avg_l4_payload_len":716,"midstream":1,"l3_proto":"ip4","src_ip":"192.168.1.12","dst_ip":"104.22.71.197","src_port":49897,"dst_port":443,"l4_proto":"tcp","ndpi": {"proto":"TLS.Cloudflare","breed":"Acceptable","category":"Web"},"tls": {"version":"TLSv1.2","client_requested_server_name":"","ja3":"e5ef852e686954ba9fe060fbfa881e15","ja3s":"","unsafe_cipher":0,"cipher":"TLS_NULL_WITH_NULL_NULL","alpn":"h2,http\/1.1","tls_supported_versions":"TLSv1.3,TLSv1.2,TLSv1.1,TLSv1"}}
-00507{"flow_event_id":3,"flow_event_name":"idle","thread_id":0,"packet_id":3,"source":"encrypted_sni.pcap","alias":"nDPId-test","flow_id":1,"flow_packet_id":1,"flow_first_seen":1590680386576,"flow_last_seen":1590680386576,"flow_min_l4_payload_len":716,"flow_max_l4_payload_len":716,"flow_tot_l4_payload_len":716,"flow_avg_l4_payload_len":716,"midstream":1,"l3_proto":"ip4","src_ip":"192.168.1.12","dst_ip":"104.27.129.77","src_port":49886,"dst_port":443,"l4_proto":"tcp","flow_datalink":1,"flow_max_packets":15}
-00507{"flow_event_id":3,"flow_event_name":"idle","thread_id":0,"packet_id":3,"source":"encrypted_sni.pcap","alias":"nDPId-test","flow_id":3,"flow_packet_id":1,"flow_first_seen":1590680391590,"flow_last_seen":1590680391590,"flow_min_l4_payload_len":716,"flow_max_l4_payload_len":716,"flow_tot_l4_payload_len":716,"flow_avg_l4_payload_len":716,"midstream":1,"l3_proto":"ip4","src_ip":"192.168.1.12","dst_ip":"104.22.71.197","src_port":49897,"dst_port":443,"l4_proto":"tcp","flow_datalink":1,"flow_max_packets":15}
-00508{"flow_event_id":3,"flow_event_name":"idle","thread_id":0,"packet_id":3,"source":"encrypted_sni.pcap","alias":"nDPId-test","flow_id":2,"flow_packet_id":1,"flow_first_seen":1590680387847,"flow_last_seen":1590680387847,"flow_min_l4_payload_len":716,"flow_max_l4_payload_len":716,"flow_tot_l4_payload_len":716,"flow_avg_l4_payload_len":716,"midstream":1,"l3_proto":"ip4","src_ip":"192.168.1.12","dst_ip":"104.16.125.175","src_port":49887,"dst_port":443,"l4_proto":"tcp","flow_datalink":1,"flow_max_packets":15}
-00132{"daemon_event_id":3,"daemon_event_name":"shutdown","thread_id":0,"packet_id":3,"source":"encrypted_sni.pcap","alias":"nDPId-test"}
+00447{"daemon_event_id":1,"daemon_event_name":"init","thread_id":0,"packet_id":0,"source":"encrypted_sni.pcap","alias":"nDPId-test","max-flows-per-thread":2048,"max-idle-flows-per-thread":256,"tick-resolution":1000,"reader-thread-count":1,"flow-scan-interval":10000,"generic-max-idle-time":600000,"icmp-max-idle-time":120000,"udp-max-idle-time":180000,"tcp-max-idle-time":7460000,"max-packets-per-flow-to-send":3,"max-packets-per-flow-to-process":255}
+00562{"flow_event_id":1,"flow_event_name":"new","thread_id":0,"packet_id":1,"source":"encrypted_sni.pcap","alias":"nDPId-test","flow_id":1,"flow_packets_processed":1,"flow_first_seen":1590680386576,"flow_last_seen":1590680386576,"flow_idle_time":7440000,"flow_min_l4_payload_len":716,"flow_max_l4_payload_len":716,"flow_tot_l4_payload_len":716,"flow_avg_l4_payload_len":716,"midstream":1,"ts_msec":1590680386576,"l3_proto":"ip4","src_ip":"192.168.1.12","dst_ip":"104.27.129.77","src_port":49886,"dst_port":443,"l4_proto":"tcp","flow_datalink":1,"flow_max_packets":3}
+01415{"packet_event_id":2,"packet_event_name":"packet-flow","thread_id":0,"packet_id":1,"source":"encrypted_sni.pcap","alias":"nDPId-test","flow_id":1,"flow_packet_id":1,"flow_last_seen":1590680386576,"flow_idle_time":7440000,"pkt_oversize":false,"pkt_caplen":770,"pkt_type":2048,"pkt_l3_offset":14,"pkt_l4_offset":34,"pkt_len":770,"pkt_l4_len":736,"ts_msec":1590680386576,"pkt":"EBMx8Tl2KDc3AG3ICABFAAL0AABAAEAGjOfAqAEMaBuBTcLeAbt3Q5LX\/48DFVAYIACwHgAAFgMBAscBAALDAwOTwM86TEdZaYZx77QiKeLaOUyI6FPS+J3L+0S3MA31OCDtrXy2AkmiC5EC8aXH8NKs5TG5ofTGvlsmIWUcTFlOhgAkEwETAxMCwCvAL8ypzKjALMAwwArACcATwBQAMwA5AC8ANQAKAQACVgAXAAD\/AQABAAAKAA4ADAAdABcAGAAZAQABAQALAAIBAAAjAAAAEAAOAAwCaDIIaHR0cC8xLjEABQAFAQAAAAAAMwBrAGkAHQAg9C+VXLX0pUAYcvwRMlm2BfjMFL+A2Ha+teHeYm8XszAAFwBBBKhP+5j\/iIqKULsVEv1xkLdgIoxwczB5EVKfTq\/0aLaIOqqUx255GoGIKzaHGdYeWvgG2FTscntynOjMKiH+1xMAKwAJCAMEAwMDAgMBAA0AGAAWBAMFAwYDCAQIBQgGBAEFAQYBAgMCAQAtAAIBAf\/OAW4TAQAdACAoJey8d6KdccaSJO2lCYt20kw0EEYFyldVNE\/b+wVlLQAgHyQSymUyoBaYNvGbjOJlOzPcW4r7yiRdTxErCb+vUsgBJJYkyzxOIwgn94z1v2QNIt6jP8xZjqajLZOZBVhvvpl7nmhmH4lW1IkwcuGd4kzR+4ip9x\/EzAG6tckU\/flqZH1nG16JhZuu6rEiIYaISW303wwyjD1flAsQnOsqJ0PVy+NZQoiiKbjH4viDA+P+GiaonlAB8r2TaJD+948G4F7MBjpovbjBjfrBFM8f7NuL4fwv7ssjFdJ5mNaCsSn9Hj6115hdy9xFKhCCzMA44L9pVw\/vrGvG+5UfibZ5LK2nZAPALOtdzhzm7d0W1ff7a4XSuSSFRI3gCI5CHoPx4osmf747Wa4ElvuEUhPCcdTFrF6efl9qMHJEUwf8zrcwZxBFmZHEDMTcH8MlFUx5dN14A3E5eAVFahmuI+6IR1wd8HaXtmYAHAACQAE="}
+00847{"flow_event_id":6,"flow_event_name":"detected","thread_id":0,"packet_id":1,"source":"encrypted_sni.pcap","alias":"nDPId-test","flow_id":1,"flow_packets_processed":1,"flow_first_seen":1590680386576,"flow_last_seen":1590680386576,"flow_idle_time":7440000,"flow_min_l4_payload_len":716,"flow_max_l4_payload_len":716,"flow_tot_l4_payload_len":716,"flow_avg_l4_payload_len":716,"midstream":1,"ts_msec":1590680386576,"l3_proto":"ip4","src_ip":"192.168.1.12","dst_ip":"104.27.129.77","src_port":49886,"dst_port":443,"l4_proto":"tcp","ndpi": {"proto":"TLS.Cloudflare","breed":"Acceptable","category":"Web"},"tls": {"version":"TLSv1.2","client_requested_server_name":"","ja3":"e5ef852e686954ba9fe060fbfa881e15","ja3s":"","unsafe_cipher":0,"cipher":"TLS_NULL_WITH_NULL_NULL","alpn":"h2,http\/1.1","tls_supported_versions":"TLSv1.3,TLSv1.2,TLSv1.1,TLSv1"}}
+00563{"flow_event_id":1,"flow_event_name":"new","thread_id":0,"packet_id":2,"source":"encrypted_sni.pcap","alias":"nDPId-test","flow_id":2,"flow_packets_processed":1,"flow_first_seen":1590680387847,"flow_last_seen":1590680387847,"flow_idle_time":7440000,"flow_min_l4_payload_len":716,"flow_max_l4_payload_len":716,"flow_tot_l4_payload_len":716,"flow_avg_l4_payload_len":716,"midstream":1,"ts_msec":1590680387847,"l3_proto":"ip4","src_ip":"192.168.1.12","dst_ip":"104.16.125.175","src_port":49887,"dst_port":443,"l4_proto":"tcp","flow_datalink":1,"flow_max_packets":3}
+01419{"packet_event_id":2,"packet_event_name":"packet-flow","thread_id":0,"packet_id":2,"source":"encrypted_sni.pcap","alias":"nDPId-test","flow_id":2,"flow_packet_id":1,"flow_last_seen":1590680387847,"flow_idle_time":7440000,"pkt_oversize":false,"pkt_caplen":770,"pkt_type":2048,"pkt_l3_offset":14,"pkt_l4_offset":34,"pkt_len":770,"pkt_l4_len":736,"ts_msec":1590680387847,"pkt":"EBMx8Tl2KDc3AG3ICABFAAL0AABAAEAGkJDAqAEMaBB9r8LfAbu98X4VZuCG7lAYIACqfgAAFgMBAscBAALDAwPZvt6xqK7JiSO2eRBioUk2Uu867QdPWpn6Sv4hYS472iAz8c+AKNafKEsBeorsjdYMXk2HdHvKJL23Af8gga\/qxAAkEwETAxMCwCvAL8ypzKjALMAwwArACcATwBQAMwA5AC8ANQAKAQACVgAXAAD\/AQABAAAKAA4ADAAdABcAGAAZAQABAQALAAIBAAAjAAAAEAAOAAwCaDIIaHR0cC8xLjEABQAFAQAAAAAAMwBrAGkAHQAg0HCVKAanlLS9J1B8hdchDfkoKDxcPc3B5hBZYsZWdz8AFwBBBCakAur\/e3rF+tGl0au7NOTY4DQpBg\/YjV6ew74w8otvaCGiCdoeWGhEGjsldqwZrBxN3o59i8BSdRX+YPQ+GgkAKwAJCAMEAwMDAgMBAA0AGAAWBAMFAwYDCAQIBQgGBAEFAQYBAgMCAQAtAAIBAf\/OAW4TAQAdACAFyK2kXV21yqtAW2T62b\/NDTnJgxOrhECle3qcjynhZQAgHyQSymUyoBaYNvGbjOJlOzPcW4r7yiRdTxErCb+vUsgBJLkAAE456EuY9a6HsKAg7En+2G8rSItqsoven5V2IfJ3Q2bekOZcTKgIZokRYkaF7ExtxsFhqXy+gigbwIQnaXqjvmpA5fAKz4tj4ykxew5OhWQtUKuHkOYZfaYtn1syOdzFlDd5f+dopSDJ1HH+q6E3XfYeSjmwk2PLEJ57JKeThEiW3dFrbufb5XbXZxYdeC179v7EU6Bakj2Njpvv\/Jfo5WxPGqtw\/pm8l4GeHZCKXzswlPS\/Jet6JKlP28PhB6QjuLs0HyKQD3u9h3gOMLbs85P+uPv\/61THn6BnP+Gq0XsiHUv\/ZFCqDNSvUTBmtmCAtgIUfzrLcUWkNsVonaILrLi\/m6vYUQElVuyPe7nXS\/qvJdz0NipXdWB8POXCwp8YOWkAHAACQAE="}
+00848{"flow_event_id":6,"flow_event_name":"detected","thread_id":0,"packet_id":2,"source":"encrypted_sni.pcap","alias":"nDPId-test","flow_id":2,"flow_packets_processed":1,"flow_first_seen":1590680387847,"flow_last_seen":1590680387847,"flow_idle_time":7440000,"flow_min_l4_payload_len":716,"flow_max_l4_payload_len":716,"flow_tot_l4_payload_len":716,"flow_avg_l4_payload_len":716,"midstream":1,"ts_msec":1590680387847,"l3_proto":"ip4","src_ip":"192.168.1.12","dst_ip":"104.16.125.175","src_port":49887,"dst_port":443,"l4_proto":"tcp","ndpi": {"proto":"TLS.Cloudflare","breed":"Acceptable","category":"Web"},"tls": {"version":"TLSv1.2","client_requested_server_name":"","ja3":"e5ef852e686954ba9fe060fbfa881e15","ja3s":"","unsafe_cipher":0,"cipher":"TLS_NULL_WITH_NULL_NULL","alpn":"h2,http\/1.1","tls_supported_versions":"TLSv1.3,TLSv1.2,TLSv1.1,TLSv1"}}
+00562{"flow_event_id":1,"flow_event_name":"new","thread_id":0,"packet_id":3,"source":"encrypted_sni.pcap","alias":"nDPId-test","flow_id":3,"flow_packets_processed":1,"flow_first_seen":1590680391590,"flow_last_seen":1590680391590,"flow_idle_time":7440000,"flow_min_l4_payload_len":716,"flow_max_l4_payload_len":716,"flow_tot_l4_payload_len":716,"flow_avg_l4_payload_len":716,"midstream":1,"ts_msec":1590680391590,"l3_proto":"ip4","src_ip":"192.168.1.12","dst_ip":"104.22.71.197","src_port":49897,"dst_port":443,"l4_proto":"tcp","flow_datalink":1,"flow_max_packets":3}
+01413{"packet_event_id":2,"packet_event_name":"packet-flow","thread_id":0,"packet_id":3,"source":"encrypted_sni.pcap","alias":"nDPId-test","flow_id":3,"flow_packet_id":1,"flow_last_seen":1590680391590,"flow_idle_time":7440000,"pkt_oversize":false,"pkt_caplen":770,"pkt_type":2048,"pkt_l3_offset":14,"pkt_l4_offset":34,"pkt_len":770,"pkt_l4_len":736,"ts_msec":1590680391590,"pkt":"EBMx8Tl2KDc3AG3ICABFAAL0AABAAEAGxnTAqAEMaBZHxcLpAbsLJg40SW6gUlAYIAANXgAAFgMBAscBAALDAwMJLl9l\/OldUJYbpqd0xOpts3Kv4zg2hroTXcdX9KeB2CBjkfBVUTqX532YPuVZHQd0J5lIK2OZH9nsSRBnWwKDWwAkEwETAxMCwCvAL8ypzKjALMAwwArACcATwBQAMwA5AC8ANQAKAQACVgAXAAD\/AQABAAAKAA4ADAAdABcAGAAZAQABAQALAAIBAAAjAAAAEAAOAAwCaDIIaHR0cC8xLjEABQAFAQAAAAAAMwBrAGkAHQAgsbxhJX9IcnjB7rdgEb2YIBohnnxEhKIToNk1er8CIioAFwBBBLtlLNXLCuP0okhISXwuyj6tgeyLGZ5yaSZ9uT3zAbum2y5l1gYjS6RGBBL9dNcuY2pA4Ze582sOuuo0cAvw2TsAKwAJCAMEAwMDAgMBAA0AGAAWBAMFAwYDCAQIBQgGBAEFAQYBAgMCAQAtAAIBAf\/OAW4TAQAdACCgcq\/jSZGFwhXJHl9nfU84W9RHblecX+XHXi+knd++egAgHyQSymUyoBaYNvGbjOJlOzPcW4r7yiRdTxErCb+vUsgBJM1prHJ\/+qDqcKEqpG5xU365kjS5loGMkTxyoKwRhL+l3TthfgE+TKCSsunPt4vNjTPLrxKpdN+3jkm4v5pXmXQY7xTIeDCWHjyEgNKkvyfWHZEc70MAkkqfNhBXSLrthF\/1heQEBlRbs1xtqteJZDPsTf1rb0lyjahdcH23rHhPVaZljcat4wh7Hka7vt+kTz6HVLMaa8+FGdKR02KYBfqCbkN5nqbjMCHPCoPKBXF7APN9aYQZNPW1vyVMZGeIilksOKMAfbO31cu423QrZX+PlzwFC6qBeqVxOTzYpLwLIxJGCnfdBRD0u85D1TvPM05OjHVwJVu9F3FEA\/S2klQ0zWf5b6ngXXAHdoEO61eGscgYik1z+CCLYUuTKEqAk5KVlL4AHAACQAE="}
+00847{"flow_event_id":6,"flow_event_name":"detected","thread_id":0,"packet_id":3,"source":"encrypted_sni.pcap","alias":"nDPId-test","flow_id":3,"flow_packets_processed":1,"flow_first_seen":1590680391590,"flow_last_seen":1590680391590,"flow_idle_time":7440000,"flow_min_l4_payload_len":716,"flow_max_l4_payload_len":716,"flow_tot_l4_payload_len":716,"flow_avg_l4_payload_len":716,"midstream":1,"ts_msec":1590680391590,"l3_proto":"ip4","src_ip":"192.168.1.12","dst_ip":"104.22.71.197","src_port":49897,"dst_port":443,"l4_proto":"tcp","ndpi": {"proto":"TLS.Cloudflare","breed":"Acceptable","category":"Web"},"tls": {"version":"TLSv1.2","client_requested_server_name":"","ja3":"e5ef852e686954ba9fe060fbfa881e15","ja3s":"","unsafe_cipher":0,"cipher":"TLS_NULL_WITH_NULL_NULL","alpn":"h2,http\/1.1","tls_supported_versions":"TLSv1.3,TLSv1.2,TLSv1.1,TLSv1"}}
+00563{"flow_event_id":3,"flow_event_name":"idle","thread_id":0,"packet_id":3,"source":"encrypted_sni.pcap","alias":"nDPId-test","flow_id":1,"flow_packets_processed":1,"flow_first_seen":1590680386576,"flow_last_seen":1590680386576,"flow_idle_time":7440000,"flow_min_l4_payload_len":716,"flow_max_l4_payload_len":716,"flow_tot_l4_payload_len":716,"flow_avg_l4_payload_len":716,"midstream":1,"ts_msec":1590680391590,"l3_proto":"ip4","src_ip":"192.168.1.12","dst_ip":"104.27.129.77","src_port":49886,"dst_port":443,"l4_proto":"tcp","flow_datalink":1,"flow_max_packets":3}
+00563{"flow_event_id":3,"flow_event_name":"idle","thread_id":0,"packet_id":3,"source":"encrypted_sni.pcap","alias":"nDPId-test","flow_id":3,"flow_packets_processed":1,"flow_first_seen":1590680391590,"flow_last_seen":1590680391590,"flow_idle_time":7440000,"flow_min_l4_payload_len":716,"flow_max_l4_payload_len":716,"flow_tot_l4_payload_len":716,"flow_avg_l4_payload_len":716,"midstream":1,"ts_msec":1590680391590,"l3_proto":"ip4","src_ip":"192.168.1.12","dst_ip":"104.22.71.197","src_port":49897,"dst_port":443,"l4_proto":"tcp","flow_datalink":1,"flow_max_packets":3}
+00564{"flow_event_id":3,"flow_event_name":"idle","thread_id":0,"packet_id":3,"source":"encrypted_sni.pcap","alias":"nDPId-test","flow_id":2,"flow_packets_processed":1,"flow_first_seen":1590680387847,"flow_last_seen":1590680387847,"flow_idle_time":7440000,"flow_min_l4_payload_len":716,"flow_max_l4_payload_len":716,"flow_tot_l4_payload_len":716,"flow_avg_l4_payload_len":716,"midstream":1,"ts_msec":1590680391590,"l3_proto":"ip4","src_ip":"192.168.1.12","dst_ip":"104.16.125.175","src_port":49887,"dst_port":443,"l4_proto":"tcp","flow_datalink":1,"flow_max_packets":3}
+00161{"daemon_event_id":3,"daemon_event_name":"shutdown","thread_id":0,"packet_id":3,"source":"encrypted_sni.pcap","alias":"nDPId-test","total-events-serialized":14}
 ~~~~~~~~~~~~~~~~~~~~ SUMMARY ~~~~~~~~~~~~~~~~~~~~
 ~~ packets captured/processed: 3/3
 ~~ skipped flows.............: 0
-~~ total layer4 data length..: 2208 bytes
+~~ total layer4 data length..: 2148 bytes
 ~~ total detected protocols..: 3
 ~~ total active/idle flows...: 3/3
+~~ total timeout flows.......: 0
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-~~ total memory allocated....: 1939380 bytes
-~~ total memory freed........: 1939380 bytes
+~~ total memory allocated....: 1939468 bytes
+~~ total memory freed........: 1939468 bytes
 ~~ total allocations/frees...: 35359/35359
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-~~ json string min len.......: 137 chars
-~~ json string max len.......: 1389 chars
-~~ json string avg len.......: 849 chars
+~~ json string min len.......: 166 chars
+~~ json string max len.......: 1424 chars
+~~ json string avg len.......: 866 chars