Bump libnDPI to 37f918322c0a489b5143a987c8f1a44a6f78a6f3 and updated flow json schema file.

 * export env vars AR / CMAKE_C_COMPILER_AR and RANLIB / CMAKE_C_COMPILER_RANLIB while building libnDPI
 * nDPId check API version during startup (macro vs. function call) and print a warning if they are different

Signed-off-by: Toni Uhlig <matzeton@googlemail.com>

1 files changed, 11 insertions, 10 deletions

diff --git a/test/results/psiphon3.pcap.out b/test/results/psiphon3.pcap.out
index 79e6a9411..ffc92c70c 100644
--- a/test/results/psiphon3.pcap.out
+++ b/test/results/psiphon3.pcap.out
@@ -4,11 +4,12 @@
 00458{"packet_event_id":2,"packet_event_name":"packet-flow","thread_id":0,"packet_id":1,"source":"psiphon3.pcap","alias":"nDPId-test","flow_id":1,"flow_packet_id":1,"flow_last_seen":1613865079123,"flow_idle_time":7580000,"pkt_oversize":false,"pkt_caplen":60,"pkt_type":2048,"pkt_l3_offset":0,"pkt_l4_offset":20,"pkt_len":60,"pkt_l4_len":40,"thread_ts_msec":1613865079123,"pkt":"RQAAPJ+KQABABtpRwKgAZ2gSl76ebQG7Qi4DFAAAAACgAv\/\/BPgAAAIEBbQEAggKAB2cngAAAAABAwMJ"}
 00458{"packet_event_id":2,"packet_event_name":"packet-flow","thread_id":0,"packet_id":2,"source":"psiphon3.pcap","alias":"nDPId-test","flow_id":1,"flow_packet_id":2,"flow_last_seen":1613865079129,"flow_idle_time":7580000,"pkt_oversize":false,"pkt_caplen":60,"pkt_type":2048,"pkt_l3_offset":0,"pkt_l4_offset":20,"pkt_len":60,"pkt_l4_len":40,"thread_ts_msec":1613865079129,"pkt":"RQAAPJ+KQABABtpRwKgAZ2gSl76ebQG7Qi4DFAAAAACgAv\/\/BPgAAAIEBbQEAggKAB2cngAAAAABAwMJ"}
 00451{"packet_event_id":2,"packet_event_name":"packet-flow","thread_id":0,"packet_id":3,"source":"psiphon3.pcap","alias":"nDPId-test","flow_id":1,"flow_packet_id":3,"flow_last_seen":1613865079140,"flow_idle_time":7580000,"pkt_oversize":false,"pkt_caplen":52,"pkt_type":2048,"pkt_l3_offset":0,"pkt_l4_offset":20,"pkt_len":52,"pkt_l4_len":32,"thread_ts_msec":1613865079140,"pkt":"RQAANAAAQAA8Bn3kaBKXvsCoAGcBu55t3jKOvkIuAxWAEv\/\/W\/0AAAIEBXgBAQQCAQMDCg=="}
-01078{"flow_event_id":6,"flow_event_name":"detected","thread_id":0,"packet_id":6,"source":"psiphon3.pcap","alias":"nDPId-test","flow_id":1,"flow_state":"info","flow_src_packets_processed":4,"flow_dst_packets_processed":2,"flow_first_seen":1613865079123,"flow_last_seen":1613865079143,"flow_idle_time":7580000,"flow_src_min_l4_payload_len":0,"flow_dst_min_l4_payload_len":0,"flow_src_max_l4_payload_len":168,"flow_dst_max_l4_payload_len":0,"flow_src_tot_l4_payload_len":168,"flow_dst_tot_l4_payload_len":0,"midstream":0,"thread_ts_msec":1613865079143,"l3_proto":"ip4","src_ip":"192.168.0.103","dst_ip":"104.18.151.190","src_port":40557,"dst_port":443,"l4_proto":"tcp","ndpi": {"flow_risk": {"24": {"risk":"Missing SNI TLS Extn","severity":"Medium","risk_score": {"total":500,"client":350,"server":150}}},"confidence": {"6":"DPI"},"proto":"TLS.Cloudflare","breed":"Acceptable","category":"Web"},"tls": {"version":"TLSv1.2","client_requested_server_name":"","ja3":"2d703033628575a99d44820c43b84876","ja3s":"","unsafe_cipher":0,"cipher":"TLS_NULL_WITH_NULL_NULL","alpn":"h2,http\/1.1"}}
-01141{"flow_event_id":7,"flow_event_name":"detection-update","thread_id":0,"packet_id":11,"source":"psiphon3.pcap","alias":"nDPId-test","flow_id":1,"flow_state":"info","flow_src_packets_processed":6,"flow_dst_packets_processed":5,"flow_first_seen":1613865079123,"flow_last_seen":1613865079168,"flow_idle_time":7580000,"flow_src_min_l4_payload_len":0,"flow_dst_min_l4_payload_len":0,"flow_src_max_l4_payload_len":168,"flow_dst_max_l4_payload_len":1460,"flow_src_tot_l4_payload_len":336,"flow_dst_tot_l4_payload_len":1460,"midstream":0,"thread_ts_msec":1613865079168,"l3_proto":"ip4","src_ip":"192.168.0.103","dst_ip":"104.18.151.190","src_port":40557,"dst_port":443,"l4_proto":"tcp","ndpi": {"flow_risk": {"24": {"risk":"Missing SNI TLS Extn","severity":"Medium","risk_score": {"total":500,"client":350,"server":150}}},"confidence": {"6":"DPI"},"proto":"TLS.Cloudflare","breed":"Acceptable","category":"Web"},"tls": {"version":"TLSv1.2","client_requested_server_name":"","ja3":"2d703033628575a99d44820c43b84876","ja3s":"eca9b8f0f3eae50309eaf901cb822d9b","unsafe_cipher":0,"cipher":"TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256","alpn":"h2,http\/1.1"}}
-01436{"flow_event_id":7,"flow_event_name":"detection-update","thread_id":0,"packet_id":12,"source":"psiphon3.pcap","alias":"nDPId-test","flow_id":1,"flow_state":"info","flow_src_packets_processed":6,"flow_dst_packets_processed":6,"flow_first_seen":1613865079123,"flow_last_seen":1613865079168,"flow_idle_time":7580000,"flow_src_min_l4_payload_len":0,"flow_dst_min_l4_payload_len":0,"flow_src_max_l4_payload_len":168,"flow_dst_max_l4_payload_len":1460,"flow_src_tot_l4_payload_len":336,"flow_dst_tot_l4_payload_len":2422,"midstream":0,"thread_ts_msec":1613865079168,"l3_proto":"ip4","src_ip":"192.168.0.103","dst_ip":"104.18.151.190","src_port":40557,"dst_port":443,"l4_proto":"tcp","ndpi": {"flow_risk": {"24": {"risk":"Missing SNI TLS Extn","severity":"Medium","risk_score": {"total":500,"client":350,"server":150}}},"confidence": {"6":"DPI"},"proto":"TLS.Psiphon","breed":"Acceptable","category":"VPN"},"tls": {"version":"TLSv1.2","client_requested_server_name":"","server_names":"sni.cloudflaressl.com,psiphon3.net,*.psiphon3.net","ja3":"2d703033628575a99d44820c43b84876","ja3s":"eca9b8f0f3eae50309eaf901cb822d9b","unsafe_cipher":0,"cipher":"TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256","issuerDN":"C=US, O=Cloudflare, Inc., CN=Cloudflare Inc ECC CA-3","subjectDN":"C=US, ST=CA, L=San Francisco, O=Cloudflare, Inc., CN=sni.cloudflaressl.com","alpn":"h2,http\/1.1","fingerprint":"49:30:DE:8F:B7:AF:C3:76:40:09:44:15:B4:6B:D9:8F:BE:0C:6B:0C"}}
-00936{"flow_event_id":2,"flow_event_name":"end","thread_id":0,"packet_id":62,"source":"psiphon3.pcap","alias":"nDPId-test","flow_id":1,"flow_state":"finished","flow_src_packets_processed":32,"flow_dst_packets_processed":30,"flow_first_seen":1613865079123,"flow_last_seen":1613865079845,"flow_idle_time":7580000,"flow_src_min_l4_payload_len":0,"flow_dst_min_l4_payload_len":0,"flow_src_max_l4_payload_len":1008,"flow_dst_max_l4_payload_len":1460,"flow_src_tot_l4_payload_len":3700,"flow_dst_tot_l4_payload_len":5574,"midstream":0,"thread_ts_msec":1613865079845,"l3_proto":"ip4","src_ip":"192.168.0.103","dst_ip":"104.18.151.190","src_port":40557,"dst_port":443,"l4_proto":"tcp","flow_datalink":12,"flow_max_packets":3,"ndpi": {"flow_risk": {"24": {"risk":"Missing SNI TLS Extn","severity":"Medium","risk_score": {"total":500,"client":350,"server":150}}},"confidence": {"6":"DPI"},"proto":"TLS.Psiphon","breed":"Acceptable","category":"VPN"}}
-00557{"daemon_event_id":3,"daemon_event_name":"shutdown","thread_id":0,"packet_id":62,"source":"psiphon3.pcap","alias":"nDPId-test","packets-captured":62,"packets-processed":62,"total-skipped-flows":0,"total-l4-payload-len":9274,"total-not-detected-flows":0,"total-guessed-flows":0,"total-detected-flows":1,"total-detection-updates":2,"total-updates":0,"current-active-flows":0,"total-active-flows":1,"total-idle-flows":1,"total-compressions":0,"total-compression-diff":0,"current-compression-diff":0,"total-events-serialized":11,"global_ts_msec":1613865079845}
+01108{"flow_event_id":6,"flow_event_name":"detected","thread_id":0,"packet_id":6,"source":"psiphon3.pcap","alias":"nDPId-test","flow_id":1,"flow_state":"info","flow_src_packets_processed":4,"flow_dst_packets_processed":2,"flow_first_seen":1613865079123,"flow_last_seen":1613865079143,"flow_idle_time":7580000,"flow_src_min_l4_payload_len":0,"flow_dst_min_l4_payload_len":0,"flow_src_max_l4_payload_len":168,"flow_dst_max_l4_payload_len":0,"flow_src_tot_l4_payload_len":168,"flow_dst_tot_l4_payload_len":0,"midstream":0,"thread_ts_msec":1613865079143,"l3_proto":"ip4","src_ip":"192.168.0.103","dst_ip":"104.18.151.190","src_port":40557,"dst_port":443,"l4_proto":"tcp","ndpi": {"flow_risk": {"24": {"risk":"Missing SNI TLS Extn","severity":"Medium","risk_score": {"total":500,"client":350,"server":150}}},"confidence": {"6":"DPI"},"proto":"TLS.Cloudflare","proto_id":"91.220","encrypted":1,"breed":"Acceptable","category_id":5,"category":"Web","hostname":"","tls": {"version":"TLSv1.2","ja3":"2d703033628575a99d44820c43b84876","ja3s":"","unsafe_cipher":0,"cipher":"TLS_NULL_WITH_NULL_NULL","alpn":"h2,http\/1.1"}}}
+01171{"flow_event_id":7,"flow_event_name":"detection-update","thread_id":0,"packet_id":11,"source":"psiphon3.pcap","alias":"nDPId-test","flow_id":1,"flow_state":"info","flow_src_packets_processed":6,"flow_dst_packets_processed":5,"flow_first_seen":1613865079123,"flow_last_seen":1613865079168,"flow_idle_time":7580000,"flow_src_min_l4_payload_len":0,"flow_dst_min_l4_payload_len":0,"flow_src_max_l4_payload_len":168,"flow_dst_max_l4_payload_len":1460,"flow_src_tot_l4_payload_len":336,"flow_dst_tot_l4_payload_len":1460,"midstream":0,"thread_ts_msec":1613865079168,"l3_proto":"ip4","src_ip":"192.168.0.103","dst_ip":"104.18.151.190","src_port":40557,"dst_port":443,"l4_proto":"tcp","ndpi": {"flow_risk": {"24": {"risk":"Missing SNI TLS Extn","severity":"Medium","risk_score": {"total":500,"client":350,"server":150}}},"confidence": {"6":"DPI"},"proto":"TLS.Cloudflare","proto_id":"91.220","encrypted":1,"breed":"Acceptable","category_id":5,"category":"Web","hostname":"","tls": {"version":"TLSv1.2","ja3":"2d703033628575a99d44820c43b84876","ja3s":"eca9b8f0f3eae50309eaf901cb822d9b","unsafe_cipher":0,"cipher":"TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256","alpn":"h2,http\/1.1"}}}
+01466{"flow_event_id":7,"flow_event_name":"detection-update","thread_id":0,"packet_id":12,"source":"psiphon3.pcap","alias":"nDPId-test","flow_id":1,"flow_state":"info","flow_src_packets_processed":6,"flow_dst_packets_processed":6,"flow_first_seen":1613865079123,"flow_last_seen":1613865079168,"flow_idle_time":7580000,"flow_src_min_l4_payload_len":0,"flow_dst_min_l4_payload_len":0,"flow_src_max_l4_payload_len":168,"flow_dst_max_l4_payload_len":1460,"flow_src_tot_l4_payload_len":336,"flow_dst_tot_l4_payload_len":2422,"midstream":0,"thread_ts_msec":1613865079168,"l3_proto":"ip4","src_ip":"192.168.0.103","dst_ip":"104.18.151.190","src_port":40557,"dst_port":443,"l4_proto":"tcp","ndpi": {"flow_risk": {"24": {"risk":"Missing SNI TLS Extn","severity":"Medium","risk_score": {"total":500,"client":350,"server":150}}},"confidence": {"6":"DPI"},"proto":"TLS.Psiphon","proto_id":"91.303","encrypted":1,"breed":"Acceptable","category_id":2,"category":"VPN","hostname":"","tls": {"version":"TLSv1.2","server_names":"sni.cloudflaressl.com,psiphon3.net,*.psiphon3.net","ja3":"2d703033628575a99d44820c43b84876","ja3s":"eca9b8f0f3eae50309eaf901cb822d9b","unsafe_cipher":0,"cipher":"TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256","issuerDN":"C=US, O=Cloudflare, Inc., CN=Cloudflare Inc ECC CA-3","subjectDN":"C=US, ST=CA, L=San Francisco, O=Cloudflare, Inc., CN=sni.cloudflaressl.com","alpn":"h2,http\/1.1","fingerprint":"49:30:DE:8F:B7:AF:C3:76:40:09:44:15:B4:6B:D9:8F:BE:0C:6B:0C"}}}
+01470{"flow_event_id":7,"flow_event_name":"detection-update","thread_id":0,"packet_id":32,"source":"psiphon3.pcap","alias":"nDPId-test","flow_id":1,"flow_state":"info","flow_src_packets_processed":18,"flow_dst_packets_processed":14,"flow_first_seen":1613865079123,"flow_last_seen":1613865079254,"flow_idle_time":7580000,"flow_src_min_l4_payload_len":0,"flow_dst_min_l4_payload_len":0,"flow_src_max_l4_payload_len":1008,"flow_dst_max_l4_payload_len":1460,"flow_src_tot_l4_payload_len":2038,"flow_dst_tot_l4_payload_len":5498,"midstream":0,"thread_ts_msec":1613865079254,"l3_proto":"ip4","src_ip":"192.168.0.103","dst_ip":"104.18.151.190","src_port":40557,"dst_port":443,"l4_proto":"tcp","ndpi": {"flow_risk": {"24": {"risk":"Missing SNI TLS Extn","severity":"Medium","risk_score": {"total":500,"client":350,"server":150}}},"confidence": {"6":"DPI"},"proto":"TLS.Psiphon","proto_id":"91.303","encrypted":1,"breed":"Acceptable","category_id":2,"category":"VPN","hostname":"","tls": {"version":"TLSv1.2","server_names":"sni.cloudflaressl.com,psiphon3.net,*.psiphon3.net","ja3":"2d703033628575a99d44820c43b84876","ja3s":"eca9b8f0f3eae50309eaf901cb822d9b","unsafe_cipher":0,"cipher":"TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256","issuerDN":"C=US, O=Cloudflare, Inc., CN=Cloudflare Inc ECC CA-3","subjectDN":"C=US, ST=CA, L=San Francisco, O=Cloudflare, Inc., CN=sni.cloudflaressl.com","alpn":"h2,http\/1.1","fingerprint":"49:30:DE:8F:B7:AF:C3:76:40:09:44:15:B4:6B:D9:8F:BE:0C:6B:0C"}}}
+00986{"flow_event_id":2,"flow_event_name":"end","thread_id":0,"packet_id":62,"source":"psiphon3.pcap","alias":"nDPId-test","flow_id":1,"flow_state":"finished","flow_src_packets_processed":32,"flow_dst_packets_processed":30,"flow_first_seen":1613865079123,"flow_last_seen":1613865079845,"flow_idle_time":7580000,"flow_src_min_l4_payload_len":0,"flow_dst_min_l4_payload_len":0,"flow_src_max_l4_payload_len":1008,"flow_dst_max_l4_payload_len":1460,"flow_src_tot_l4_payload_len":3700,"flow_dst_tot_l4_payload_len":5574,"midstream":0,"thread_ts_msec":1613865079845,"l3_proto":"ip4","src_ip":"192.168.0.103","dst_ip":"104.18.151.190","src_port":40557,"dst_port":443,"l4_proto":"tcp","flow_datalink":12,"flow_max_packets":3,"ndpi": {"flow_risk": {"24": {"risk":"Missing SNI TLS Extn","severity":"Medium","risk_score": {"total":500,"client":350,"server":150}}},"confidence": {"6":"DPI"},"proto":"TLS.Psiphon","proto_id":"91.303","encrypted":1,"breed":"Acceptable","category_id":2,"category":"VPN"}}
+00557{"daemon_event_id":3,"daemon_event_name":"shutdown","thread_id":0,"packet_id":62,"source":"psiphon3.pcap","alias":"nDPId-test","packets-captured":62,"packets-processed":62,"total-skipped-flows":0,"total-l4-payload-len":9274,"total-not-detected-flows":0,"total-guessed-flows":0,"total-detected-flows":1,"total-detection-updates":3,"total-updates":0,"current-active-flows":0,"total-active-flows":1,"total-idle-flows":1,"total-compressions":0,"total-compression-diff":0,"current-compression-diff":0,"total-events-serialized":12,"global_ts_msec":1613865079845}
 ~~~~~~~~~~~~~~~~~~~~ SUMMARY ~~~~~~~~~~~~~~~~~~~~
 ~~ packets captured/processed: 62/62
 ~~ skipped flows.............: 0
@@ -17,10 +18,10 @@
 ~~ total active/idle flows...: 1/1
 ~~ total timeout flows.......: 0
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-~~ total memory allocated....: 6009560 bytes
-~~ total memory freed........: 6009560 bytes
-~~ total allocations/frees...: 120947/120947
+~~ total memory allocated....: 6039519 bytes
+~~ total memory freed........: 6039519 bytes
+~~ total allocations/frees...: 121500/121500
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 ~~ json string min len.......: 456 chars
-~~ json string max len.......: 1441 chars
-~~ json string avg len.......: 921 chars
+~~ json string max len.......: 1475 chars
+~~ json string avg len.......: 951 chars