Reworked nDPIsrvd.h C-API.

 * nDPIsrvd.h: Provide nDPId thread storage.
 * nDPIsrvd.py: Fixed instance cleanup bug.
 * nDPIsrvd.h: Support for instance/thread user data and cleanup callback.
 * nDPIsrvd.h: Most recent flow time stored in thread ht instead of instance ht.
 * nDPId: Moved flow logger out the memory profilier into SIGUSR1 signal handling.
 * nDPId: Added signal fd to be usable within epoll's event handling (live-capture only!)
 * nDPId: Added information about ZLib compressions to daemon status/shutdown events.

Signed-off-by: Toni Uhlig <matzeton@googlemail.com>

1 files changed, 4 insertions, 4 deletions

diff --git a/test/results/doq.pcapng.out b/test/results/doq.pcapng.out
index 79b467c89..347311dbc 100644
--- a/test/results/doq.pcapng.out
+++ b/test/results/doq.pcapng.out
@@ -1,5 +1,5 @@
 00456{"daemon_event_id":1,"daemon_event_name":"init","thread_id":0,"packet_id":0,"source":"doq.pcapng","alias":"nDPId-test","max-flows-per-thread":2048,"max-idle-flows-per-thread":64,"tick-resolution":1000,"reader-thread-count":1,"flow-scan-interval":10000,"generic-max-idle-time":600000,"icmp-max-idle-time":120000,"udp-max-idle-time":180000,"tcp-max-idle-time":7560000,"max-packets-per-flow-to-send":3,"max-packets-per-flow-to-process":32,"global_ts_msec":0}
-00463{"daemon_event_id":4,"daemon_event_name":"status","thread_id":0,"packet_id":1,"source":"doq.pcapng","alias":"nDPId-test","packets-captured":1,"packets-processed":0,"total-skipped-flows":0,"total-l4-data-len":0,"total-not-detected-flows":0,"total-guessed-flows":0,"total-detected-flows":0,"total-detection-updates":0,"total-updates":0,"current-active-flows":0,"total-active-flows":0,"total-idle-flows":0,"total-events-serialized":2,"global_ts_msec":1606056093199}
+00542{"daemon_event_id":4,"daemon_event_name":"status","thread_id":0,"packet_id":1,"source":"doq.pcapng","alias":"nDPId-test","packets-captured":1,"packets-processed":0,"total-skipped-flows":0,"total-l4-data-len":0,"total-not-detected-flows":0,"total-guessed-flows":0,"total-detected-flows":0,"total-detection-updates":0,"total-updates":0,"current-active-flows":0,"total-active-flows":0,"total-idle-flows":0,"total-compressions":0,"total-compression-diff":0,"current-compression-diff":0,"total-events-serialized":2,"global_ts_msec":1606056093199}
 00565{"flow_event_id":1,"flow_event_name":"new","thread_id":0,"packet_id":1,"source":"doq.pcapng","alias":"nDPId-test","flow_id":1,"flow_state":"info","flow_packets_processed":1,"flow_first_seen":1606056093199,"flow_last_seen":1606056093199,"flow_idle_time":180000,"flow_min_l4_payload_len":1232,"flow_max_l4_payload_len":1232,"flow_tot_l4_payload_len":1232,"flow_avg_l4_payload_len":1232,"midstream":0,"thread_ts_msec":1606056093199,"l3_proto":"ip6","src_ip":"::1","dst_ip":"::1","src_port":47826,"dst_port":784,"l4_proto":"udp","flow_datalink":1,"flow_max_packets":3}
 02135{"packet_event_id":2,"packet_event_name":"packet-flow","thread_id":0,"packet_id":1,"source":"doq.pcapng","alias":"nDPId-test","flow_id":1,"flow_packet_id":1,"flow_last_seen":1606056093199,"flow_idle_time":180000,"pkt_oversize":false,"pkt_caplen":1294,"pkt_type":34525,"pkt_l3_offset":14,"pkt_l4_offset":54,"pkt_len":1294,"pkt_l4_len":1240,"thread_ts_msec":1606056093199,"pkt":"AAAAAAAAAAAAAAAAht1gJqqiBNgRQAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAButIDEATYBOvN\/wAAIAhwsYltsps+WghOL+O5iCYx+QBEtgiJINLAj38+CB9CqAWNGDJ\/Ht0GdZPYPfPv0gkn+G7KypaOwXpeaLqP2vrcKno6\/xJHt9kjbL2TY4b\/m9R6nztt0oBs85JJhS7Tj\/KxdnJUR5x1KoMSoiK8Up0wKQjS6CJwz+096+5cglByj68BpzrUHMPeI6GM8BR\/Wl2qjunMufbT3ODI125lDdGTaTqNLCMEIjagI12Vrkh1+4q55QnPNmDSc9uNkJ0l5bhH58Gr3GA8HfFg35RCENcGDFpWMYVXiM4ZLQRFPmW9PqqUvAkPFdK1\/e6zKceMIWl6qFwaRZM+da6dEGVcJjr7Z+tAEETRp6uqCb9nnpAvg2AYmEND50nvVEnJ0vebAvnDE4IogXJzua2gFwFm7VLYd1uL79o4iJgu\/rwI3t1+Scpc6iAB46mZWFz3fE1WDQxwSMiil9o8+U4JW1BkjaBlJjEwDLig1LbtT\/HP47m8JDRgq00wdO+B2e1saSoPUtzWH02fRpSsRwHLssxWK\/GeM8n4na9wb14wVoOdjdGJ+KEHpdBBYTSNse3PnwWrKaaP0mh7odZYLBlgeNvTBLAUy7TPWKcxmhtN6bsS\/Yjh2568CzWxz8tWmprG6YblEP1vhUU2WDKbQBSh9+e7EH2JaN6LGpgUM6\/yeDE+g\/QCDKFbnXJHaC3VNe2EpDTrUSTzTJX2ScnDPI4dI01EvvWXSfxAJzcCmkKAUz3B\/F3DS8bS2lYESb9nSox1FCQUX1S8MhWCL4jSZ4wobqLA6VEQ7puZt\/yd5mc0snO7+JferPZwSQV1jN5hdBcuNb6kj\/JG4pzUoB7QTPQcjcnBLCPQDWDzw3nQ+Ebywtgt9T0aEFqJVOTfT95bWTz6VinV\/brwfnTHpSbkUgeBvFyaDcSzRz5tFZ0q4\/gUbfajms9qKrPFsufIU5NQtKyl5gUxP+4xC0KsglyEqg4DVy8vzlOpHC9Zo8AzpD2Cd9yZUaVpS3jLxre91YlfpTBViFMhAAL1N+wl47YhA2pgyB2GGbWg1O6K4C74tiA9XM\/lrGlbtuiyqqRmlQ+OfACiiCT0\/fwnridhEP9NjW3A9LNkp5ph6u81Z1emHsIGmFkXyP7nojGy2XKkTHlNA+eKBGol\/TUgCzHu7qPwHu5vMLlk5NNq3Od8+eHViQU1LY+OXeYFHuY2S+VSf848yXn0P1WZ\/Hf4jpB8WMcPpj0cXHyY46IsajmZ4uRB40h68eDc26RMlrZAfwBIGjks8KSh5b2f1BdJ6LJ4taZkNl8x+qPVYwRdc+lJsRkcGfu+BxMBIzhOPr2wg8uauRqGpIMGiSEXt5eLhu3VHEqTuhLQrFWRwEWEm+WzY4itmVZYx3CM7zWu6j3KhN5W5HEWKe61AmbunEuzKrb9KKf1hG4Uz72IU4aUy8+qV8fLyqPe7E\/Hm\/QiosHbq0whMHw6xHc0E9dDFb7\/w2jqW\/bhRCLrrZSTu8KDShAe9bkemwaFOWgs8zleXJrozrnvcOKNBpToZAop8FcA1V6SZ+05avECZK7qQ04Uc8xlehoG+3W27ZNgeNIiTH8MtU0A5kV6veOOCPQW7GGwaBK9iuORoisN7YKGMwzzN0ZIQ\/IailJpjg=="}
 00928{"flow_event_id":6,"flow_event_name":"detected","thread_id":0,"packet_id":1,"source":"doq.pcapng","alias":"nDPId-test","flow_id":1,"flow_state":"info","flow_packets_processed":1,"flow_first_seen":1606056093199,"flow_last_seen":1606056093199,"flow_idle_time":180000,"flow_min_l4_payload_len":1232,"flow_max_l4_payload_len":1232,"flow_tot_l4_payload_len":1232,"flow_avg_l4_payload_len":1232,"midstream":0,"thread_ts_msec":1606056093199,"l3_proto":"ip6","src_ip":"::1","dst_ip":"::1","src_port":47826,"dst_port":784,"l4_proto":"udp","ndpi": {"flow_risk": {"24": {"risk":"Missing SNI TLS Extension","severity":"Medium","risk_score": {"total":500,"client":350,"server":150}}},"confidence": {"4":"DPI"},"proto":"QUIC.DoH_DoT","breed":"Fun","category":"Network"},"quic": {"version":"TLSv1.3","alpn":"doq-i00","ja3":"c0ce40fbb78cbf86a14e6a38b26d6ede","tls_supported_versions":"TLSv1.3,TLSv1.3 (draft),TLSv1.3 (draft),TLSv1.3 (draft)"}}
@@ -12,7 +12,7 @@
 00641{"packet_event_id":2,"packet_event_name":"packet-flow","thread_id":0,"packet_id":14,"source":"doq.pcapng","alias":"nDPId-test","flow_id":2,"flow_packet_id":3,"flow_last_seen":1606056093560,"flow_idle_time":120000,"pkt_oversize":false,"pkt_caplen":195,"pkt_type":34525,"pkt_l3_offset":14,"pkt_l4_offset":54,"pkt_len":195,"pkt_l4_len":141,"thread_ts_msec":1606056093560,"pkt":"AAAAAAAAAAAAAAAAht1gAryMAI06QAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAABAQQoBAAAAABgJPSDAF0RQAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAABAxC60gBdAHBcTi\/juYgmMfl+eB8WJkIN5W\/s2kV3mgzDwRAUXXe+90zefQTxG5fKyAbzm2S0iX0HuS+7+NHu2bYpwdweEdBhQ2oYMUDLzzaxqsrt98mI\/P6gjJFj"}
 00632{"flow_event_id":3,"flow_event_name":"idle","thread_id":0,"packet_id":20,"source":"doq.pcapng","alias":"nDPId-test","flow_id":2,"flow_state":"finished","flow_packets_processed":6,"flow_first_seen":1606056093260,"flow_last_seen":1606056096363,"flow_idle_time":120000,"flow_min_l4_payload_len":141,"flow_max_l4_payload_len":141,"flow_tot_l4_payload_len":846,"flow_avg_l4_payload_len":141,"midstream":0,"thread_ts_msec":1606056096363,"l3_proto":"ip6","src_ip":"::1","dst_ip":"::1","l4_proto":"icmp6","flow_datalink":1,"flow_max_packets":3,"ndpi": {"confidence": {"4":"DPI"},"proto":"ICMPV6","breed":"Acceptable","category":"Network"}}
 00795{"flow_event_id":3,"flow_event_name":"idle","thread_id":0,"packet_id":20,"source":"doq.pcapng","alias":"nDPId-test","flow_id":1,"flow_state":"finished","flow_packets_processed":14,"flow_first_seen":1606056093199,"flow_last_seen":1606056096363,"flow_idle_time":180000,"flow_min_l4_payload_len":55,"flow_max_l4_payload_len":1232,"flow_tot_l4_payload_len":3920,"flow_avg_l4_payload_len":280,"midstream":0,"thread_ts_msec":1606056096363,"l3_proto":"ip6","src_ip":"::1","dst_ip":"::1","src_port":47826,"dst_port":784,"l4_proto":"udp","flow_datalink":1,"flow_max_packets":3,"ndpi": {"flow_risk": {"24": {"risk":"Missing SNI TLS Extension","severity":"Medium","risk_score": {"total":500,"client":350,"server":150}}},"confidence": {"4":"DPI"},"proto":"QUIC.DoH_DoT","breed":"Fun","category":"Network"}}
-00472{"daemon_event_id":3,"daemon_event_name":"shutdown","thread_id":0,"packet_id":20,"source":"doq.pcapng","alias":"nDPId-test","packets-captured":20,"packets-processed":20,"total-skipped-flows":0,"total-l4-data-len":4766,"total-not-detected-flows":0,"total-guessed-flows":0,"total-detected-flows":2,"total-detection-updates":0,"total-updates":0,"current-active-flows":0,"total-active-flows":2,"total-idle-flows":2,"total-events-serialized":15,"global_ts_msec":1606056096363}
+00551{"daemon_event_id":3,"daemon_event_name":"shutdown","thread_id":0,"packet_id":20,"source":"doq.pcapng","alias":"nDPId-test","packets-captured":20,"packets-processed":20,"total-skipped-flows":0,"total-l4-data-len":4766,"total-not-detected-flows":0,"total-guessed-flows":0,"total-detected-flows":2,"total-detection-updates":0,"total-updates":0,"current-active-flows":0,"total-active-flows":2,"total-idle-flows":2,"total-compressions":0,"total-compression-diff":0,"current-compression-diff":0,"total-events-serialized":15,"global_ts_msec":1606056096363}
 ~~~~~~~~~~~~~~~~~~~~ SUMMARY ~~~~~~~~~~~~~~~~~~~~
 ~~ packets captured/processed: 20/20
 ~~ skipped flows.............: 0
@@ -21,8 +21,8 @@
 ~~ total active/idle flows...: 2/2
 ~~ total timeout flows.......: 0
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-~~ total memory allocated....: 4686702 bytes
-~~ total memory freed........: 4686702 bytes
+~~ total memory allocated....: 4686726 bytes
+~~ total memory freed........: 4686726 bytes
 ~~ total allocations/frees...: 101178/101178
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 ~~ json string min len.......: 461 chars