Refactored client distributor C API.

 * Still not perfect, but the code before was not even able to deal with JSON arrays.
   Use common "speaking" function names for all functions in nDPIsrvd.h
 * Provide a more or less generic and easy extendable JSON walk function.
 * Modified C examples to align with the changed C API.
 * c-collectd: Reduced lot's of code duplication by providing mapping tables.
 * nDPId: IAT array requires one slot less (first packet has always an IAT of 0).

Signed-off-by: Toni Uhlig <matzeton@googlemail.com>

1 files changed, 3 insertions, 3 deletions

diff --git a/test/results/android.pcap.out b/test/results/android.pcap.out
index 01f443f0b..1b2fcb2f9 100644
--- a/test/results/android.pcap.out
+++ b/test/results/android.pcap.out
@@ -297,7 +297,7 @@
 00550{"packet_event_id":2,"packet_event_name":"packet-flow","thread_id":0,"packet_id":406,"source":"android.pcap","alias":"nDPId-test","flow_id":60,"flow_packet_id":1,"flow_src_last_pkt_time":1582454871881494,"flow_dst_last_pkt_time":1582454871881494,"flow_idle_time":200000000,"pkt_oversize":false,"pkt_caplen":82,"pkt_type":2048,"pkt_l3_offset":14,"pkt_l4_offset":34,"pkt_len":82,"pkt_l4_len":48,"thread_ts_usec":1582454871881494,"pkt":"xiwDYGpkTGr2n\/YnCABFAABErDBAAEARCRfAqAIQwKgCAZtQADUAMNjjuKUBAAABAAAAAAAAB2FuZHJvaWQKZ29vZ2xlYXBpcwNjb20AAAEAAQ=="}
 01013{"flow_event_id":7,"flow_event_name":"detected","thread_id":0,"packet_id":406,"source":"android.pcap","alias":"nDPId-test","flow_id":60,"flow_state":"info","flow_src_packets_processed":1,"flow_dst_packets_processed":0,"flow_first_seen":1582454871881494,"flow_src_last_pkt_time":1582454871881494,"flow_dst_last_pkt_time":1582454871881494,"flow_idle_time":200000000,"flow_src_min_l4_payload_len":40,"flow_dst_min_l4_payload_len":0,"flow_src_max_l4_payload_len":40,"flow_dst_max_l4_payload_len":0,"flow_src_tot_l4_payload_len":40,"flow_dst_tot_l4_payload_len":0,"midstream":0,"thread_ts_usec":1582454871881494,"l3_proto":"ip4","src_ip":"192.168.2.16","dst_ip":"192.168.2.1","src_port":39760,"dst_port":53,"l4_proto":"udp","ndpi": {"confidence": {"6":"DPI"},"proto":"DNS.GoogleServices","proto_id":"5.239","encrypted":0,"breed":"Acceptable","category_id":5,"category":"Web","hostname":"android.googleapis.com","dns": {"num_queries":0,"num_answers":0,"reply_code":0,"query_type":1,"rsp_type":0,"rsp_addr":"0.0.0.0"}}}
 01117{"flow_event_id":7,"flow_event_name":"detected","thread_id":0,"packet_id":408,"source":"android.pcap","alias":"nDPId-test","flow_id":58,"flow_state":"info","flow_src_packets_processed":3,"flow_dst_packets_processed":1,"flow_first_seen":1582454871829800,"flow_src_last_pkt_time":1582454871890562,"flow_dst_last_pkt_time":1582454871867294,"flow_idle_time":3285032704,"flow_src_min_l4_payload_len":0,"flow_dst_min_l4_payload_len":0,"flow_src_max_l4_payload_len":517,"flow_dst_max_l4_payload_len":0,"flow_src_tot_l4_payload_len":517,"flow_dst_tot_l4_payload_len":0,"midstream":0,"thread_ts_usec":1582454871890562,"l3_proto":"ip4","src_ip":"192.168.2.16","dst_ip":"172.217.20.76","src_port":43646,"dst_port":443,"l4_proto":"tcp","ndpi": {"confidence": {"6":"DPI"},"proto":"TLS.DataSaver","proto_id":"91.46","encrypted":1,"breed":"Fun","category_id":5,"category":"Web","hostname":"proxy.googlezip.net","tls": {"version":"TLSv1.2","ja3":"66918128f1b9b03303d77c6f2eefd128","ja3s":"","unsafe_cipher":0,"cipher":"TLS_NULL_WITH_NULL_NULL","alpn":"h2,http\/1.1","tls_supported_versions":"GREASE,TLSv1.3,TLSv1.2,TLSv1.1,TLSv1"}}}
-01718{"flow_event_id":5,"flow_event_name":"analyse","thread_id":0,"packet_id":431,"source":"android.pcap","alias":"nDPId-test","flow_id":42,"flow_state":"finished","flow_src_packets_processed":17,"flow_dst_packets_processed":15,"flow_first_seen":1582454871152402,"flow_src_last_pkt_time":1582454871906464,"flow_dst_last_pkt_time":1582454871901421,"flow_idle_time":3285032704,"flow_src_min_l4_payload_len":0,"flow_dst_min_l4_payload_len":0,"flow_src_max_l4_payload_len":512,"flow_dst_max_l4_payload_len":1418,"flow_src_tot_l4_payload_len":819,"flow_dst_tot_l4_payload_len":10828,"midstream":0,"thread_ts_usec":1582454871906464,"l3_proto":"ip4","src_ip":"192.168.2.16","dst_ip":"216.239.38.120","src_port":32996,"dst_port":443,"l4_proto":"tcp","flow_datalink":1,"flow_max_packets":3,"data_analysis": {"iat": {"min":3,"avg":48486.5,"max":404574,"stddev":104241.1,"var":10866214912.0,"ent":3.0,"data": [13673,15022,32725,47474,16568,3,34518,282,386517,404574,19668,197623,221096,19209,15019,27735,41804,1657,22,36,1002,1575,133,18,9,1204,14,1169,2703,19,10,0]},"pktlen": {"min":66,"avg":430.5,"max":1484,"stddev":552.7,"var":305506.2,"ent":4.0,"data": [74,74,66,246,66,1484,1202,66,66,159,358,66,578,66,100,66,655,66,1484,1484,1421,1484,66,1484,396,102,66,66,66,66,66,66]},"bins": {"c_to_s": [13,1,1,0,0,1,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"s_to_c": [4,1,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,5,0,0,0]},"directions": [0,1,0,0,1,1,1,0,0,0,1,0,0,1,0,1,1,0,1,1,1,1,0,1,1,1,0,0,0,0,0,0]},"ndpi": {"confidence": {"6":"DPI"},"proto":"TLS.Google","proto_id":"91.126","encrypted":1,"breed":"Acceptable","category_id":5,"category":"Web"}}
+01716{"flow_event_id":5,"flow_event_name":"analyse","thread_id":0,"packet_id":431,"source":"android.pcap","alias":"nDPId-test","flow_id":42,"flow_state":"finished","flow_src_packets_processed":17,"flow_dst_packets_processed":15,"flow_first_seen":1582454871152402,"flow_src_last_pkt_time":1582454871906464,"flow_dst_last_pkt_time":1582454871901421,"flow_idle_time":3285032704,"flow_src_min_l4_payload_len":0,"flow_dst_min_l4_payload_len":0,"flow_src_max_l4_payload_len":512,"flow_dst_max_l4_payload_len":1418,"flow_src_tot_l4_payload_len":819,"flow_dst_tot_l4_payload_len":10828,"midstream":0,"thread_ts_usec":1582454871906464,"l3_proto":"ip4","src_ip":"192.168.2.16","dst_ip":"216.239.38.120","src_port":32996,"dst_port":443,"l4_proto":"tcp","flow_datalink":1,"flow_max_packets":3,"data_analysis": {"iat": {"min":3,"avg":48486.5,"max":404574,"stddev":104241.1,"var":10866214912.0,"ent":3.0,"data": [13673,15022,32725,47474,16568,3,34518,282,386517,404574,19668,197623,221096,19209,15019,27735,41804,1657,22,36,1002,1575,133,18,9,1204,14,1169,2703,19,10]},"pktlen": {"min":66,"avg":430.5,"max":1484,"stddev":552.7,"var":305506.2,"ent":4.0,"data": [74,74,66,246,66,1484,1202,66,66,159,358,66,578,66,100,66,655,66,1484,1484,1421,1484,66,1484,396,102,66,66,66,66,66,66]},"bins": {"c_to_s": [13,1,1,0,0,1,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"s_to_c": [4,1,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,5,0,0,0]},"directions": [0,1,0,0,1,1,1,0,0,0,1,0,0,1,0,1,1,0,1,1,1,1,0,1,1,1,0,0,0,0,0,0]},"ndpi": {"confidence": {"6":"DPI"},"proto":"TLS.Google","proto_id":"91.126","encrypted":1,"breed":"Acceptable","category_id":5,"category":"Web"}}
 01163{"flow_event_id":8,"flow_event_name":"detection-update","thread_id":0,"packet_id":434,"source":"android.pcap","alias":"nDPId-test","flow_id":59,"flow_state":"info","flow_src_packets_processed":3,"flow_dst_packets_processed":3,"flow_first_seen":1582454871839297,"flow_src_last_pkt_time":1582454871880409,"flow_dst_last_pkt_time":1582454871911317,"flow_idle_time":3285032704,"flow_src_min_l4_payload_len":0,"flow_dst_min_l4_payload_len":0,"flow_src_max_l4_payload_len":517,"flow_dst_max_l4_payload_len":1418,"flow_src_tot_l4_payload_len":517,"flow_dst_tot_l4_payload_len":1418,"midstream":0,"thread_ts_usec":1582454871911317,"l3_proto":"ip4","src_ip":"192.168.2.16","dst_ip":"216.239.38.120","src_port":33014,"dst_port":443,"l4_proto":"tcp","ndpi": {"confidence": {"6":"DPI"},"proto":"TLS.Google","proto_id":"91.126","encrypted":1,"breed":"Acceptable","category_id":5,"category":"Web","hostname":"www.google.com","tls": {"version":"TLSv1.3","ja3":"66918128f1b9b03303d77c6f2eefd128","ja3s":"eb1d94daa7e0344597e756a1fb6e7054","unsafe_cipher":0,"cipher":"TLS_AES_128_GCM_SHA256","alpn":"h2,http\/1.1","tls_supported_versions":"GREASE,TLSv1.3,TLSv1.2,TLSv1.1,TLSv1"}}}
 01166{"flow_event_id":8,"flow_event_name":"detection-update","thread_id":0,"packet_id":437,"source":"android.pcap","alias":"nDPId-test","flow_id":55,"flow_state":"info","flow_src_packets_processed":3,"flow_dst_packets_processed":3,"flow_first_seen":1582454871814833,"flow_src_last_pkt_time":1582454871879681,"flow_dst_last_pkt_time":1582454871913572,"flow_idle_time":3285032704,"flow_src_min_l4_payload_len":0,"flow_dst_min_l4_payload_len":0,"flow_src_max_l4_payload_len":594,"flow_dst_max_l4_payload_len":212,"flow_src_tot_l4_payload_len":594,"flow_dst_tot_l4_payload_len":212,"midstream":0,"thread_ts_usec":1582454871913572,"l3_proto":"ip4","src_ip":"192.168.2.16","dst_ip":"172.217.21.202","src_port":51944,"dst_port":443,"l4_proto":"tcp","ndpi": {"confidence": {"6":"DPI"},"proto":"TLS.DataSaver","proto_id":"91.46","encrypted":1,"breed":"Fun","category_id":5,"category":"Web","hostname":"datasaver.googleapis.com","tls": {"version":"TLSv1.3","ja3":"554719594ba90b02ae410c297c6e50ad","ja3s":"2b0648ab686ee45e0e7c35fcfb0eea7e","unsafe_cipher":0,"cipher":"TLS_AES_128_GCM_SHA256","alpn":"h2,http\/1.1","tls_supported_versions":"GREASE,TLSv1.3,TLSv1.2,TLSv1.1,TLSv1"}}}
 00570{"packet_event_id":2,"packet_event_name":"packet-flow","thread_id":0,"packet_id":441,"source":"android.pcap","alias":"nDPId-test","flow_id":60,"flow_packet_id":2,"flow_src_last_pkt_time":1582454871881494,"flow_dst_last_pkt_time":1582454871920611,"flow_idle_time":200000000,"pkt_oversize":false,"pkt_caplen":98,"pkt_type":2048,"pkt_l3_offset":14,"pkt_l4_offset":34,"pkt_len":98,"pkt_l4_len":64,"thread_ts_usec":1582454871920611,"pkt":"TGr2n\/YnxiwDYGpkCABFAABUFXQAAEAR38PAqAIBwKgCEAA1m1AAQNQ0uKWBgAABAAEAAAAAB2FuZHJvaWQKZ29vZ2xlYXBpcwNjb20AAAEAAcAMAAEAAQAAARcABKzZFgo="}
@@ -392,8 +392,8 @@
 ~~ total active/idle flows...: 63/63
 ~~ total timeout flows.......: 0
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-~~ total memory allocated....: 6392603 bytes
-~~ total memory freed........: 6392603 bytes
+~~ total memory allocated....: 6392351 bytes
+~~ total memory freed........: 6392351 bytes
 ~~ total allocations/frees...: 122858/122858
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 ~~ json string min len.......: 492 chars