Reworked nDPIsrvd.h C-API.

 * nDPIsrvd.h: Provide nDPId thread storage.
 * nDPIsrvd.py: Fixed instance cleanup bug.
 * nDPIsrvd.h: Support for instance/thread user data and cleanup callback.
 * nDPIsrvd.h: Most recent flow time stored in thread ht instead of instance ht.
 * nDPId: Moved flow logger out the memory profilier into SIGUSR1 signal handling.
 * nDPId: Added signal fd to be usable within epoll's event handling (live-capture only!)
 * nDPId: Added information about ZLib compressions to daemon status/shutdown events.

Signed-off-by: Toni Uhlig <matzeton@googlemail.com>

1 files changed, 4 insertions, 4 deletions

diff --git a/test/results/tls-rdn-extract.pcap.out b/test/results/tls-rdn-extract.pcap.out
index c61061874..97a55d806 100644
--- a/test/results/tls-rdn-extract.pcap.out
+++ b/test/results/tls-rdn-extract.pcap.out
@@ -1,5 +1,5 @@
 00466{"daemon_event_id":1,"daemon_event_name":"init","thread_id":0,"packet_id":0,"source":"tls-rdn-extract.pcap","alias":"nDPId-test","max-flows-per-thread":2048,"max-idle-flows-per-thread":64,"tick-resolution":1000,"reader-thread-count":1,"flow-scan-interval":10000,"generic-max-idle-time":600000,"icmp-max-idle-time":120000,"udp-max-idle-time":180000,"tcp-max-idle-time":7560000,"max-packets-per-flow-to-send":3,"max-packets-per-flow-to-process":32,"global_ts_msec":0}
-00472{"daemon_event_id":4,"daemon_event_name":"status","thread_id":0,"packet_id":1,"source":"tls-rdn-extract.pcap","alias":"nDPId-test","packets-captured":1,"packets-processed":0,"total-skipped-flows":0,"total-l4-data-len":0,"total-not-detected-flows":0,"total-guessed-flows":0,"total-detected-flows":0,"total-detection-updates":0,"total-updates":0,"current-active-flows":0,"total-active-flows":0,"total-idle-flows":0,"total-events-serialized":2,"global_ts_msec":946681200000}
+00551{"daemon_event_id":4,"daemon_event_name":"status","thread_id":0,"packet_id":1,"source":"tls-rdn-extract.pcap","alias":"nDPId-test","packets-captured":1,"packets-processed":0,"total-skipped-flows":0,"total-l4-data-len":0,"total-not-detected-flows":0,"total-guessed-flows":0,"total-detected-flows":0,"total-detection-updates":0,"total-updates":0,"current-active-flows":0,"total-active-flows":0,"total-idle-flows":0,"total-compressions":0,"total-compression-diff":0,"current-compression-diff":0,"total-events-serialized":2,"global_ts_msec":946681200000}
 00586{"flow_event_id":1,"flow_event_name":"new","thread_id":0,"packet_id":1,"source":"tls-rdn-extract.pcap","alias":"nDPId-test","flow_id":1,"flow_state":"info","flow_packets_processed":1,"flow_first_seen":946681200000,"flow_last_seen":946681200000,"flow_idle_time":7440000,"flow_min_l4_payload_len":127,"flow_max_l4_payload_len":127,"flow_tot_l4_payload_len":127,"flow_avg_l4_payload_len":127,"midstream":1,"thread_ts_msec":946681200000,"l3_proto":"ip4","src_ip":"10.0.0.1","dst_ip":"213.199.149.251","src_port":31337,"dst_port":443,"l4_proto":"tcp","flow_datalink":1,"flow_max_packets":3}
 00631{"packet_event_id":2,"packet_event_name":"packet-flow","thread_id":0,"packet_id":1,"source":"tls-rdn-extract.pcap","alias":"nDPId-test","flow_id":1,"flow_packet_id":1,"flow_last_seen":946681200000,"flow_idle_time":7440000,"pkt_oversize":false,"pkt_caplen":181,"pkt_type":2048,"pkt_l3_offset":14,"pkt_l4_offset":34,"pkt_len":181,"pkt_l4_len":147,"thread_ts_msec":946681200000,"pkt":"ERERERERIiIiIiIiCABFAACnLudAAIAGnZoKAAAB1ceV+3ppAbtkZ4Ye79i2a1AYQCmgXgAAFgMBAHoBAAB2AwEAAAAAM7RDB2u\/HXE+9PsbFMYgy+4A2s6CH4THeQytZwAAGAAvADUABQAKwBPAFMAJwAoAMgA4ABMABAEAADX\/AQABAAAAABMAEQAADmFkczEubXNhZHMubmV0AAUABQEAAAAAAAoABgAEABcAGAALAAIBAA=="}
 00951{"flow_event_id":6,"flow_event_name":"detected","thread_id":0,"packet_id":1,"source":"tls-rdn-extract.pcap","alias":"nDPId-test","flow_id":1,"flow_state":"info","flow_packets_processed":1,"flow_first_seen":946681200000,"flow_last_seen":946681200000,"flow_idle_time":7440000,"flow_min_l4_payload_len":127,"flow_max_l4_payload_len":127,"flow_tot_l4_payload_len":127,"flow_avg_l4_payload_len":127,"midstream":1,"thread_ts_msec":946681200000,"l3_proto":"ip4","src_ip":"10.0.0.1","dst_ip":"213.199.149.251","src_port":31337,"dst_port":443,"l4_proto":"tcp","ndpi": {"flow_risk": {"7": {"risk":"Obsolete TLS Version (1.1 or older)","severity":"High","risk_score": {"total":510,"client":455,"server":55}}},"confidence": {"4":"DPI"},"proto":"TLS","breed":"Safe","category":"Web"},"tls": {"version":"TLSv1","client_requested_server_name":"ads1.msads.net","ja3":"2201d8e006f8f005a6b415f61e677532","ja3s":"","unsafe_cipher":0,"cipher":"TLS_NULL_WITH_NULL_NULL"}}
@@ -8,7 +8,7 @@
 02411{"packet_event_id":2,"packet_event_name":"packet-flow","thread_id":0,"packet_id":3,"source":"tls-rdn-extract.pcap","alias":"nDPId-test","flow_id":1,"flow_packet_id":3,"flow_last_seen":946681200000,"flow_idle_time":7440000,"pkt_oversize":false,"pkt_caplen":1514,"pkt_type":2048,"pkt_l3_offset":14,"pkt_l4_offset":34,"pkt_len":1514,"pkt_l4_len":1480,"thread_ts_msec":946681200000,"pkt":"ERERERERIiIiIiIiCABFAAXc5PNAADUGLVnVx5X7CgAAAQG7emnv2LwfZGeGnVAQGJjDXgAAbTEUMBIGA1UEAwwLKi5zLW1zbi5jb20xFzAVBgNVBAMMDioubGl2ZS1pbnQubmV0MR8wHQYDVQQDDBYqLndpbmRvd3NwaG9uZS1pbnQuY29tMRswGQYDVQQDDBIqLndpbmRvd3NwaG9uZS5jb20xKjAoBgNVBAMMISoucGFydG5lci1wYy53aW5kb3dzcGhvbmUtaW50LmNvbTEfMB0GA1UEAwwWKi5tYW5hZ2UubWljcm9zb2Z0LmNvbTEYMBYGA1UEAwwPKi52by5tc2VjbmQubmV0MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAuX3PkoiInBfw68+6JNH406C4alrEnikcq1FZEZJZj8A0h7uDLWO01R+9CYljtZsYv4E+pfWvi8Z31QoN\/mqJYHgutax6\/UWMDIxFsXaIn1iXAoBA481Pyqa8XbzdmibAvotkEOm0ksJYJlu7VrGuQP+fyz69HW2nTnewmEyTsEy9pTZjqsxFdtBcWm2sS5KQA3Hoj6NzWl54VkXacUcpgQraZZFiSKVJpxhZpAqND3x7NCgSdQvwN2uTFwRCsRagxmCSSaZkQSbYCDh7lvCo6r5wBODibkMqCxrJ4nyg5Uw+J74SsSHhtBMkb6YMlWe5gPOyYSZfIVCby4onZWx45wIDAQABo4IGXzCCBlswDAYDVR0TAQH\/BAIwADALBgNVHQ8EBAMCBLAwHQYDVR0lBBYwFAYIKwYBBQUHAwIGCCsGAQUFBwMBMIIDowYDVR0RBIIDmjCCA5aCDyoudm8ubXNlY25kLm5ldIIVKi5vZmZpY2VhcHBzLmxpdmUuY29tggsqLm1zYWRzLm5ldIIQKi5hZHMyLm1zYWRzLm5ldIIPKi5zdGMucy1tc24uY29tgiRjZG4uZGMyZmlsZXMuKi5saXZlZmlsZXN0b3JlLWludC5jb22CF2Nkbi4qLmxpdmVmaWxlc3RvcmUuY29tgh8qLm1hcmtldHBsYWNlLndpbmRvd3Ntb2JpbGUuY29tgiMqLm1hcmtldHBsYWNlLndpbmRvd3Ntb2JpbGUtaW50LmNvbYIkKi5tYXJrZXRwbGFjZS53aW5kb3dzbW9iaWxlLXBlcmYuY29tgg8qLnN0ai5zLW1zbi5jb22CEmFqYXgubWljcm9zb2Z0LmNvbYIbKi5taWNyb3NvZnQtc2JzLWRvbWFpbnMuY29tggoqLmxpdmUubmV0ggkqLm1zbi5jb22CDSoubXNuLWludC5jb22CGiouZjFkcy5zaGFyZWQubGl2ZS1pbnQuY29tghQqLmYxZHMud2x4cnMtaW50LmNvbYIVKi5zaGFyZWQubGl2ZS1pbnQuY29tghEqLnNoYXJlZC5saXZlLmNvbYIPKi5taWNyb3NvZnQuY29tggoqLmxpdmUuY29tgg4qLmxpdmUtaW50LmNvbYILKi53bHhycy5jb22CDyoud2x4cnMtaW50LmNvbYIOKi5zdC5zLW1zbi5jb22CDyouc3RiLnMtbXNuLmNvbYIgaW1hZ2VzLm1veHkud2luZG93c3Bob25lLWludC5jb22CECoud2x4cnN1LWludC5jb22CI2ltYWdlcy5wYXJ0bmVyLndpbmRvd3NwaG9uZS1pbnQuY29tgh9pbWFnZXMucGFydG5lci53aW5kb3dzcGhvbmUuY29tggwqLmpwLm1zbi5jb22CEiouYzNzY3MuanAubXNuLmNvbYIPKi5hc3BuZXRjZG4uY29tgg0qLmhvdG1haWwuY29tgiEqLnBhcnRuZXItZGYud2luZG93c3Bob25lLWludC5jb22CCyoucy1tc24uY29tgg4qLmxpdmUtaW50Lm5ldIIWKi53aW5kb3dzcGhvbmUtaW50LmNvbYISKi53aW5kb3dzcGhvbmUuY29tgiEqLnBhcnRuZXI="}
 03337{"flow_event_id":7,"flow_event_name":"detection-update","thread_id":0,"packet_id":6,"source":"tls-rdn-extract.pcap","alias":"nDPId-test","flow_id":1,"flow_state":"info","flow_packets_processed":6,"flow_first_seen":946681200000,"flow_last_seen":946681200000,"flow_idle_time":7440000,"flow_min_l4_payload_len":127,"flow_max_l4_payload_len":1460,"flow_tot_l4_payload_len":6881,"flow_avg_l4_payload_len":1146,"midstream":1,"thread_ts_msec":946681200000,"l3_proto":"ip4","src_ip":"10.0.0.1","dst_ip":"213.199.149.251","src_port":31337,"dst_port":443,"l4_proto":"tcp","ndpi": {"flow_risk": {"7": {"risk":"Obsolete TLS Version (1.1 or older)","severity":"High","risk_score": {"total":510,"client":455,"server":55}},"8": {"risk":"Weak TLS Cipher","severity":"High","risk_score": {"total":250,"client":225,"server":25}},"9": {"risk":"TLS Expired Certificate","severity":"High","risk_score": {"total":260,"client":230,"server":30}}},"confidence": {"4":"DPI"},"proto":"TLS.Microsoft","breed":"Safe","category":"Web"},"tls": {"version":"TLSv1","client_requested_server_name":"ads1.msads.net","server_names":"*.vo.msecnd.net,*.officeapps.live.com,*.msads.net,*.ads2.msads.net,*.stc.s-msn.com,cdn.dc2files.*.livefilestore-int.com,cdn.*.livefilestore.com,*.marketplace.windowsmobile.com,*.marketplace.windowsmobile-int.com,*.marketplace.windowsmobile-perf.com,*.stj.s-msn.com,ajax.microsoft.com,*.microsoft-sbs-domains.com,*.live.net,*.msn.com,*.msn-int.com,*.f1ds.shared.live-int.com,*.f1ds.wlxrs-int.com,*.shared.live-int.com,*.shared.live.com,*.microsoft.com,*.live.com,*.live-int.com,*.wlxrs.com,*.wlxrs-int.com,*.st.s-msn.com,*.stb.s-msn.com,images.moxy.windowsphone-int.com,*.wlxrsu-int.com,images.partner.windowsphone-int.com,images.partner.windowsphone.com,*.jp.msn.com,*.c3scs.jp.msn.com,*.aspnetcdn.com,*.hotmail.com,*.partner-df.windowsphone-int.com,*.s-msn.com,*.live-int.net,*.windowsphone-int.com,*.windowsphone.com,*.partner-pc.windowsphone-int.com,*.manage.microsoft.com","ja3":"2201d8e006f8f005a6b415f61e677532","ja3s":"18e962e106761869a61045bed0e81c2c","unsafe_cipher":1,"cipher":"TLS_RSA_WITH_AES_128_CBC_SHA","issuerDN":"CN=Microsoft Secure Server Authority","subjectDN":"C=US, L=Redmond, O=Microsoft, OU=GFS, CN=*.officeapps.live.com, CN=*.msads.net, CN=*.ads2.msads.net, CN=*.stc.s-msn.com, CN=cdn.dc2files.*.livefilestore-int.com, CN=cdn.*.livefilestore.com, CN=*.marketplace.windowsmobile.com, CN=*.marketplace.windowsmobile-int.com, CN=*.marketplace.windowsmobile-perf.com, CN=*.stj.s-msn.com, CN=ajax.microsoft.com, CN=*.microsoft-sbs-domains.com, CN=*.live.net, CN=*.msn.com, CN=*.msn-int.com, CN=*.f1ds.shared.live-int.com, CN=*.f1ds.wlxrs-int.com, CN=*.shared.live-int.com, CN=*.shared.live.com, CN=*.microsoft.com, CN=*.live.com, CN=*.live-int.com, CN=*.wlxrs.com, CN=*.wlxrs-int.com, CN=*.st.s-msn.com, CN=*.stb.s-msn.com, CN=images.moxy.windowsphone-int.com, CN=*.wlxrsu-int.com, CN=images.partner.windowsphone-int.com, CN=images.partner.windowsphone.com, CN=*.jp.msn.com, CN=*.c3scs.jp.msn.com, CN=*.aspnetcdn.com, CN=*.hotmail.com, CN=*.partner-df.windowsphone-int.com, CN=*.s-msn.com, CN=*.live-int.net, CN=*.windowsphone-int.com, CN=*.windowsphone.com, CN=*.partner-pc.windowsphone-int.com, CN=*.manage.microsoft.com, CN=*.vo.msecnd.net","fingerprint":"FF:BF:9A:69:8F:C8:44:FF:89:F2:61:49:A7:D1:9A:98:DE:32:84:3B"}}
 01038{"flow_event_id":3,"flow_event_name":"idle","thread_id":0,"packet_id":6,"source":"tls-rdn-extract.pcap","alias":"nDPId-test","flow_id":1,"flow_state":"finished","flow_packets_processed":6,"flow_first_seen":946681200000,"flow_last_seen":946681200000,"flow_idle_time":7440000,"flow_min_l4_payload_len":127,"flow_max_l4_payload_len":1460,"flow_tot_l4_payload_len":6881,"flow_avg_l4_payload_len":1146,"midstream":1,"thread_ts_msec":946681200000,"l3_proto":"ip4","src_ip":"10.0.0.1","dst_ip":"213.199.149.251","src_port":31337,"dst_port":443,"l4_proto":"tcp","flow_datalink":1,"flow_max_packets":3,"ndpi": {"flow_risk": {"7": {"risk":"Obsolete TLS Version (1.1 or older)","severity":"High","risk_score": {"total":510,"client":455,"server":55}},"8": {"risk":"Weak TLS Cipher","severity":"High","risk_score": {"total":250,"client":225,"server":25}},"9": {"risk":"TLS Expired Certificate","severity":"High","risk_score": {"total":260,"client":230,"server":30}}},"confidence": {"4":"DPI"},"proto":"TLS.Microsoft","breed":"Safe","category":"Web"}}
-00478{"daemon_event_id":3,"daemon_event_name":"shutdown","thread_id":0,"packet_id":6,"source":"tls-rdn-extract.pcap","alias":"nDPId-test","packets-captured":6,"packets-processed":6,"total-skipped-flows":0,"total-l4-data-len":6881,"total-not-detected-flows":0,"total-guessed-flows":0,"total-detected-flows":1,"total-detection-updates":2,"total-updates":0,"current-active-flows":0,"total-active-flows":1,"total-idle-flows":1,"total-events-serialized":11,"global_ts_msec":946681200000}
+00557{"daemon_event_id":3,"daemon_event_name":"shutdown","thread_id":0,"packet_id":6,"source":"tls-rdn-extract.pcap","alias":"nDPId-test","packets-captured":6,"packets-processed":6,"total-skipped-flows":0,"total-l4-data-len":6881,"total-not-detected-flows":0,"total-guessed-flows":0,"total-detected-flows":1,"total-detection-updates":2,"total-updates":0,"current-active-flows":0,"total-active-flows":1,"total-idle-flows":1,"total-compressions":0,"total-compression-diff":0,"current-compression-diff":0,"total-events-serialized":11,"global_ts_msec":946681200000}
 ~~~~~~~~~~~~~~~~~~~~ SUMMARY ~~~~~~~~~~~~~~~~~~~~
 ~~ packets captured/processed: 6/6
 ~~ skipped flows.............: 0
@@ -17,8 +17,8 @@
 ~~ total active/idle flows...: 1/1
 ~~ total timeout flows.......: 0
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-~~ total memory allocated....: 4721639 bytes
-~~ total memory freed........: 4721639 bytes
+~~ total memory allocated....: 4721663 bytes
+~~ total memory freed........: 4721663 bytes
 ~~ total allocations/frees...: 101198/101198
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 ~~ json string min len.......: 471 chars