Refactored client distributor C API.

 * Still not perfect, but the code before was not even able to deal with JSON arrays.
   Use common "speaking" function names for all functions in nDPIsrvd.h
 * Provide a more or less generic and easy extendable JSON walk function.
 * Modified C examples to align with the changed C API.
 * c-collectd: Reduced lot's of code duplication by providing mapping tables.
 * nDPId: IAT array requires one slot less (first packet has always an IAT of 0).

Signed-off-by: Toni Uhlig <matzeton@googlemail.com>

1 files changed, 5 insertions, 5 deletions

diff --git a/test/results/dns_exfiltration.pcap.out b/test/results/dns_exfiltration.pcap.out
index 3053b44dd..0b481905a 100644
--- a/test/results/dns_exfiltration.pcap.out
+++ b/test/results/dns_exfiltration.pcap.out
@@ -6,7 +6,7 @@
 00963{"packet_event_id":2,"packet_event_name":"packet-flow","thread_id":0,"packet_id":2,"source":"dns_exfiltration.pcap","alias":"nDPId-test","flow_id":1,"flow_packet_id":2,"flow_src_last_pkt_time":1580978146717893,"flow_dst_last_pkt_time":1580978146888524,"flow_idle_time":200000000,"pkt_oversize":false,"pkt_caplen":386,"pkt_type":2048,"pkt_l3_offset":14,"pkt_l4_offset":34,"pkt_len":386,"pkt_l4_len":352,"thread_ts_usec":1580978146888524,"pkt":"jNzURr7Eqqru7hERCABFAAF0PC1AAD8R1RrAqMunwKjcOAA13DUBYD3xOR2BgAABAAEAAAAABmRuc2NhdDw1NDZiMDNmNTAwMDAwMDAwMDBhNjAyM2VkNGRmMTg0ZDZhYzVjMjYyOGI0NzcxNGZkZWU1ODRmZWQ3Mzk8NWEwM2I1YjFlMWFhOGY4ZmRiMWJiZThkNWUwNDk1MjE0MWY3ZDRmODJjN2UzYjA2ZGNjOGI4N2ZhZDdhGjE5ZTRkMDk4ZGM4YzYxOGY4ZDgxY2ZlYjAyAAAPAAHADAAPAAEAAAA8AJ8ACgZkbnNjYXQ\/MjAxZjAzZjUwMDAwMDAwMDAwNzEzYjkyNzFmMDExZGM3NjQyM2RhYjM5MmMzMmMxOGJmYzk2YjZkMjY5NWEyPzZhOTExYzk0NDcyZjU5NDA5YTVmNTI2MDEzZTc2MDE5MzY2YTA3NzkyOWUzNDgwZmJlNmQ3YzRlZGE2ZjkwOBRmMmJjOTlhNjAxZTFhODIyMTMzNgA="}
 01325{"flow_event_id":8,"flow_event_name":"detection-update","thread_id":0,"packet_id":2,"source":"dns_exfiltration.pcap","alias":"nDPId-test","flow_id":1,"flow_state":"info","flow_src_packets_processed":1,"flow_dst_packets_processed":1,"flow_first_seen":1580978146717893,"flow_src_last_pkt_time":1580978146717893,"flow_dst_last_pkt_time":1580978146888524,"flow_idle_time":200000000,"flow_src_min_l4_payload_len":173,"flow_dst_min_l4_payload_len":0,"flow_src_max_l4_payload_len":173,"flow_dst_max_l4_payload_len":344,"flow_src_tot_l4_payload_len":173,"flow_dst_tot_l4_payload_len":344,"midstream":0,"thread_ts_usec":1580978146888524,"l3_proto":"ip4","src_ip":"192.168.220.56","dst_ip":"192.168.203.167","src_port":56373,"dst_port":53,"l4_proto":"udp","ndpi": {"flow_risk": {"16": {"risk":"Suspicious DGA Domain name","severity":"High","risk_score": {"total":250,"client":125,"server":125}},"27": {"risk":"Risky Domain Name","severity":"Medium","risk_score": {"total":760,"client":580,"server":180}}},"confidence": {"6":"DPI"},"proto":"DNS","proto_id":"5","encrypted":0,"breed":"Acceptable","category_id":14,"category":"Network","hostname":"e1aa8f8fdb1bbe8d5e04952141f7d4f82c7e3b06dcc8b87fad7a.19e4d098dc8c618f8d81cfeb02","dns": {"num_queries":1,"num_answers":1,"reply_code":0,"query_type":15,"rsp_type":15,"rsp_addr":"0.0.0.0"}}}
 00670{"packet_event_id":2,"packet_event_name":"packet-flow","thread_id":0,"packet_id":3,"source":"dns_exfiltration.pcap","alias":"nDPId-test","flow_id":1,"flow_packet_id":3,"flow_src_last_pkt_time":1580978147753419,"flow_dst_last_pkt_time":1580978146888524,"flow_idle_time":200000000,"pkt_oversize":false,"pkt_caplen":166,"pkt_type":2048,"pkt_l3_offset":14,"pkt_l4_offset":34,"pkt_len":166,"pkt_l4_len":132,"thread_ts_usec":1580978147753419,"pkt":"qqru7hERjNzURr7ECABFAACYekZAAD8RAADAqNw4wKjLp9w1ADUAhCnHfRoBAAABAAAAAAAABmRuc2NhdDw5MWYwMDNmNTAwZjYxMjIxODEwYWVhMDAwMDA0ODYzYzY5MTU4MGVjYWQ2NmY2NGFjN2RkYjg3Yjg5YzcmOTIwMDgyMWU1MjdkNGUxNzYzMjUzYzI1ZTI5N2UyYWE0MTEzZDAAAAUAAQ=="}
-02058{"flow_event_id":5,"flow_event_name":"analyse","thread_id":0,"packet_id":32,"source":"dns_exfiltration.pcap","alias":"nDPId-test","flow_id":1,"flow_state":"finished","flow_src_packets_processed":16,"flow_dst_packets_processed":16,"flow_first_seen":1580978146717893,"flow_src_last_pkt_time":1580978160880828,"flow_dst_last_pkt_time":1580978160882236,"flow_idle_time":200000000,"flow_src_min_l4_payload_len":59,"flow_dst_min_l4_payload_len":0,"flow_src_max_l4_payload_len":173,"flow_dst_max_l4_payload_len":344,"flow_src_tot_l4_payload_len":1158,"flow_dst_tot_l4_payload_len":2183,"midstream":0,"thread_ts_usec":1580978160882236,"l3_proto":"ip4","src_ip":"192.168.220.56","dst_ip":"192.168.203.167","src_port":56373,"dst_port":53,"l4_proto":"udp","flow_datalink":1,"flow_max_packets":3,"data_analysis": {"iat": {"min":3976,"avg":913783.2,"max":1035526,"stddev":281798.4,"var":79410348032.0,"ent":4.8,"data": [170631,1035526,866477,1015270,1015599,4647,3976,1009971,1010376,1009201,1009121,1008475,1008435,1009499,1009380,1008042,1008120,1008655,1008570,1009773,1009797,1009990,1010112,1008960,1008939,1008465,1008353,1007666,1007763,1008795,1008694,0]},"pktlen": {"min":101,"avg":146.4,"max":386,"stddev":59.1,"var":3497.9,"ent":4.9,"data": [215,386,166,286,136,193,101,148,101,148,101,156,101,148,101,158,101,158,101,156,101,148,101,158,101,158,101,158,101,148,101,148]},"bins": {"c_to_s": [0,13,1,1,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"s_to_c": [0,0,0,13,1,0,0,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]},"directions": [0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1]},"ndpi": {"flow_risk": {"16": {"risk":"Suspicious DGA Domain name","severity":"High","risk_score": {"total":250,"client":125,"server":125}},"27": {"risk":"Risky Domain Name","severity":"Medium","risk_score": {"total":760,"client":580,"server":180}}},"confidence": {"6":"DPI"},"proto":"DNS","proto_id":"5","encrypted":0,"breed":"Acceptable","category_id":14,"category":"Network"}}
+02056{"flow_event_id":5,"flow_event_name":"analyse","thread_id":0,"packet_id":32,"source":"dns_exfiltration.pcap","alias":"nDPId-test","flow_id":1,"flow_state":"finished","flow_src_packets_processed":16,"flow_dst_packets_processed":16,"flow_first_seen":1580978146717893,"flow_src_last_pkt_time":1580978160880828,"flow_dst_last_pkt_time":1580978160882236,"flow_idle_time":200000000,"flow_src_min_l4_payload_len":59,"flow_dst_min_l4_payload_len":0,"flow_src_max_l4_payload_len":173,"flow_dst_max_l4_payload_len":344,"flow_src_tot_l4_payload_len":1158,"flow_dst_tot_l4_payload_len":2183,"midstream":0,"thread_ts_usec":1580978160882236,"l3_proto":"ip4","src_ip":"192.168.220.56","dst_ip":"192.168.203.167","src_port":56373,"dst_port":53,"l4_proto":"udp","flow_datalink":1,"flow_max_packets":3,"data_analysis": {"iat": {"min":3976,"avg":913783.2,"max":1035526,"stddev":281798.4,"var":79410348032.0,"ent":4.8,"data": [170631,1035526,866477,1015270,1015599,4647,3976,1009971,1010376,1009201,1009121,1008475,1008435,1009499,1009380,1008042,1008120,1008655,1008570,1009773,1009797,1009990,1010112,1008960,1008939,1008465,1008353,1007666,1007763,1008795,1008694]},"pktlen": {"min":101,"avg":146.4,"max":386,"stddev":59.1,"var":3497.9,"ent":4.9,"data": [215,386,166,286,136,193,101,148,101,148,101,156,101,148,101,158,101,158,101,156,101,148,101,158,101,158,101,158,101,148,101,148]},"bins": {"c_to_s": [0,13,1,1,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"s_to_c": [0,0,0,13,1,0,0,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]},"directions": [0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1]},"ndpi": {"flow_risk": {"16": {"risk":"Suspicious DGA Domain name","severity":"High","risk_score": {"total":250,"client":125,"server":125}},"27": {"risk":"Risky Domain Name","severity":"Medium","risk_score": {"total":760,"client":580,"server":180}}},"confidence": {"6":"DPI"},"proto":"DNS","proto_id":"5","encrypted":0,"breed":"Acceptable","category_id":14,"category":"Network"}}
 01163{"flow_event_id":4,"flow_event_name":"update","thread_id":0,"packet_id":115,"source":"dns_exfiltration.pcap","alias":"nDPId-test","flow_id":1,"flow_state":"finished","flow_src_packets_processed":57,"flow_dst_packets_processed":57,"flow_first_seen":1580978146717893,"flow_src_last_pkt_time":1580978196387731,"flow_dst_last_pkt_time":1580978196389199,"flow_idle_time":200000000,"flow_src_min_l4_payload_len":59,"flow_dst_min_l4_payload_len":0,"flow_src_max_l4_payload_len":258,"flow_dst_max_l4_payload_len":344,"flow_src_tot_l4_payload_len":4115,"flow_dst_tot_l4_payload_len":7851,"midstream":0,"thread_ts_usec":1580978196389199,"l3_proto":"ip4","src_ip":"192.168.220.56","dst_ip":"192.168.203.167","src_port":56373,"dst_port":53,"l4_proto":"udp","flow_datalink":1,"flow_max_packets":3,"ndpi": {"flow_risk": {"16": {"risk":"Suspicious DGA Domain name","severity":"High","risk_score": {"total":250,"client":125,"server":125}},"27": {"risk":"Risky Domain Name","severity":"Medium","risk_score": {"total":760,"client":580,"server":180}}},"confidence": {"6":"DPI"},"proto":"DNS","proto_id":"5","encrypted":0,"breed":"Acceptable","category_id":14,"category":"Network"}}
 01165{"flow_event_id":3,"flow_event_name":"idle","thread_id":0,"packet_id":300,"source":"dns_exfiltration.pcap","alias":"nDPId-test","flow_id":1,"flow_state":"finished","flow_src_packets_processed":150,"flow_dst_packets_processed":150,"flow_first_seen":1580978146717893,"flow_src_last_pkt_time":1580978206706247,"flow_dst_last_pkt_time":1580978206707432,"flow_idle_time":200000000,"flow_src_min_l4_payload_len":59,"flow_dst_min_l4_payload_len":0,"flow_src_max_l4_payload_len":258,"flow_dst_max_l4_payload_len":344,"flow_src_tot_l4_payload_len":26119,"flow_dst_tot_l4_payload_len":34826,"midstream":0,"thread_ts_usec":1580978206707432,"l3_proto":"ip4","src_ip":"192.168.220.56","dst_ip":"192.168.203.167","src_port":56373,"dst_port":53,"l4_proto":"udp","flow_datalink":1,"flow_max_packets":3,"ndpi": {"flow_risk": {"16": {"risk":"Suspicious DGA Domain name","severity":"High","risk_score": {"total":250,"client":125,"server":125}},"27": {"risk":"Risky Domain Name","severity":"Medium","risk_score": {"total":760,"client":580,"server":180}}},"confidence": {"6":"DPI"},"proto":"DNS","proto_id":"5","encrypted":0,"breed":"Acceptable","category_id":14,"category":"Network"}}
 00572{"daemon_event_id":3,"daemon_event_name":"shutdown","thread_id":0,"packet_id":300,"source":"dns_exfiltration.pcap","alias":"nDPId-test","packets-captured":300,"packets-processed":300,"total-skipped-flows":0,"total-l4-payload-len":60945,"total-not-detected-flows":0,"total-guessed-flows":0,"total-detected-flows":1,"total-detection-updates":1,"total-updates":1,"current-active-flows":0,"total-active-flows":1,"total-idle-flows":1,"total-compressions":0,"total-compression-diff":0,"current-compression-diff":0,"total-events-serialized":12,"global_ts_usec":1580978206707432}
@@ -18,10 +18,10 @@
 ~~ total active/idle flows...: 1/1
 ~~ total timeout flows.......: 0
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-~~ total memory allocated....: 6044459 bytes
-~~ total memory freed........: 6044459 bytes
+~~ total memory allocated....: 6044455 bytes
+~~ total memory freed........: 6044455 bytes
 ~~ total allocations/frees...: 121789/121789
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 ~~ json string min len.......: 501 chars
-~~ json string max len.......: 2063 chars
-~~ json string avg len.......: 1257 chars
+~~ json string max len.......: 2061 chars
+~~ json string avg len.......: 1256 chars