Refactored client distributor C API.

 * Still not perfect, but the code before was not even able to deal with JSON arrays.
   Use common "speaking" function names for all functions in nDPIsrvd.h
 * Provide a more or less generic and easy extendable JSON walk function.
 * Modified C examples to align with the changed C API.
 * c-collectd: Reduced lot's of code duplication by providing mapping tables.
 * nDPId: IAT array requires one slot less (first packet has always an IAT of 0).

Signed-off-by: Toni Uhlig <matzeton@googlemail.com>

1 files changed, 4 insertions, 4 deletions

diff --git a/test/results/trickbot.pcap.out b/test/results/trickbot.pcap.out
index 9b2e97e2c..ca5beed09 100644
--- a/test/results/trickbot.pcap.out
+++ b/test/results/trickbot.pcap.out
@@ -6,7 +6,7 @@
 00510{"packet_event_id":2,"packet_event_name":"packet-flow","thread_id":0,"packet_id":3,"source":"trickbot.pcap","alias":"nDPId-test","flow_id":1,"flow_packet_id":3,"flow_src_last_pkt_time":1609266107797418,"flow_dst_last_pkt_time":1609266107797175,"flow_idle_time":3285032704,"pkt_oversize":false,"pkt_caplen":54,"pkt_type":2048,"pkt_l3_offset":14,"pkt_l4_offset":34,"pkt_len":54,"pkt_l4_len":20,"thread_ts_usec":1609266107797418,"pkt":"IOUqtpPxAAgCHEeuCABFAAAoc9JAAIAGK1IKDB1lUnbhxO+GG6gSdtdXYu1SXVAQ\/\/+p4QAA"}
 01412{"flow_event_id":7,"flow_event_name":"detected","thread_id":0,"packet_id":4,"source":"trickbot.pcap","alias":"nDPId-test","flow_id":1,"flow_state":"info","flow_src_packets_processed":3,"flow_dst_packets_processed":1,"flow_first_seen":1609266107551500,"flow_src_last_pkt_time":1609266107797621,"flow_dst_last_pkt_time":1609266107797175,"flow_idle_time":3285032704,"flow_src_min_l4_payload_len":0,"flow_dst_min_l4_payload_len":0,"flow_src_max_l4_payload_len":349,"flow_dst_max_l4_payload_len":0,"flow_src_tot_l4_payload_len":349,"flow_dst_tot_l4_payload_len":0,"midstream":0,"thread_ts_usec":1609266107797621,"l3_proto":"ip4","src_ip":"10.12.29.101","dst_ip":"82.118.225.196","src_port":61318,"dst_port":7080,"l4_proto":"tcp","ndpi": {"flow_risk": {"5": {"risk":"Known Proto on Non Std Port","severity":"Medium","risk_score": {"total":260,"client":230,"server":30}},"12": {"risk":"HTTP Numeric IP Address","severity":"Low","risk_score": {"total":500,"client":450,"server":50}}},"confidence": {"6":"DPI"},"proto":"HTTP","proto_id":"7","encrypted":0,"breed":"Acceptable","category_id":5,"category":"Web","hostname":"82.118.225.196","http": {"url":"82.118.225.196:7080\/OK21pqJAtyyGBEo00sk","code":0,"content_type":"","user_agent":"Mozilla\/4.0 (compatible; MSIE 7.0; Windows NT 10.0; WOW64; Trident\/7.0; .NET4.0C; .NET4.0E)","request_content_type":"application\/x-www-form-urlencoded","detected_os":"Windows 10"}}}
 01552{"flow_event_id":8,"flow_event_name":"detection-update","thread_id":0,"packet_id":8,"source":"trickbot.pcap","alias":"nDPId-test","flow_id":1,"flow_state":"info","flow_src_packets_processed":4,"flow_dst_packets_processed":4,"flow_first_seen":1609266107551500,"flow_src_last_pkt_time":1609266107797702,"flow_dst_last_pkt_time":1609266108728827,"flow_idle_time":3285032704,"flow_src_min_l4_payload_len":0,"flow_dst_min_l4_payload_len":0,"flow_src_max_l4_payload_len":928,"flow_dst_max_l4_payload_len":1358,"flow_src_tot_l4_payload_len":1277,"flow_dst_tot_l4_payload_len":1358,"midstream":0,"thread_ts_usec":1609266108728827,"l3_proto":"ip4","src_ip":"10.12.29.101","dst_ip":"82.118.225.196","src_port":61318,"dst_port":7080,"l4_proto":"tcp","ndpi": {"flow_risk": {"5": {"risk":"Known Proto on Non Std Port","severity":"Medium","risk_score": {"total":260,"client":230,"server":30}},"12": {"risk":"HTTP Numeric IP Address","severity":"Low","risk_score": {"total":500,"client":450,"server":50}},"25": {"risk":"HTTP Suspicious Content","severity":"High","risk_score": {"total":510,"client":355,"server":155}}},"confidence": {"6":"DPI"},"proto":"HTTP","proto_id":"7","encrypted":0,"breed":"Acceptable","category_id":5,"category":"Web","hostname":"82.118.225.196","http": {"url":"82.118.225.196:7080\/OK21pqJAtyyGBEo00sk","code":200,"content_type":"text\/html","user_agent":"Mozilla\/4.0 (compatible; MSIE 7.0; Windows NT 10.0; WOW64; Trident\/7.0; .NET4.0C; .NET4.0E)","request_content_type":"application\/x-www-form-urlencoded","detected_os":"Windows 10"}}}
-02062{"flow_event_id":5,"flow_event_name":"analyse","thread_id":0,"packet_id":32,"source":"trickbot.pcap","alias":"nDPId-test","flow_id":1,"flow_state":"finished","flow_src_packets_processed":9,"flow_dst_packets_processed":23,"flow_first_seen":1609266107551500,"flow_src_last_pkt_time":1609266109737227,"flow_dst_last_pkt_time":1609266110219915,"flow_idle_time":3285032704,"flow_src_min_l4_payload_len":0,"flow_dst_min_l4_payload_len":0,"flow_src_max_l4_payload_len":928,"flow_dst_max_l4_payload_len":1460,"flow_src_tot_l4_payload_len":1277,"flow_dst_tot_l4_payload_len":27187,"midstream":0,"thread_ts_usec":1609266110219915,"l3_proto":"ip4","src_ip":"10.12.29.101","dst_ip":"82.118.225.196","src_port":61318,"dst_port":7080,"l4_proto":"tcp","flow_datalink":1,"flow_max_packets":3,"data_analysis": {"iat": {"min":6,"avg":156585.2,"max":931328,"stddev":258444.3,"var":66793451520.0,"ent":3.3,"data": [245675,245918,203,81,530,37,931085,931328,2339,2280,480234,19,480300,297566,15,8,7,8,7,8,8,7,7,6,9,297680,227938,227937,482874,14,14,0]},"pktlen": {"min":54,"avg":944.0,"max":1514,"stddev":662.5,"var":438885.5,"ent":4.5,"data": [66,58,54,403,982,54,54,1412,54,1412,54,1514,1337,54,1514,1514,1514,1514,1514,1514,1514,1514,1514,1514,1514,290,54,1412,54,1514,1514,1208]},"bins": {"c_to_s": [7,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"s_to_c": [3,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,1,0,3,0,0,14,0,0]},"directions": [0,1,0,0,0,1,1,1,0,1,0,1,1,0,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,1,1,1]},"ndpi": {"flow_risk": {"5": {"risk":"Known Proto on Non Std Port","severity":"Medium","risk_score": {"total":260,"client":230,"server":30}},"12": {"risk":"HTTP Numeric IP Address","severity":"Low","risk_score": {"total":500,"client":450,"server":50}},"25": {"risk":"HTTP Suspicious Content","severity":"High","risk_score": {"total":510,"client":355,"server":155}}},"confidence": {"6":"DPI"},"proto":"HTTP","proto_id":"7","encrypted":0,"breed":"Acceptable","category_id":5,"category":"Web"}}
+02060{"flow_event_id":5,"flow_event_name":"analyse","thread_id":0,"packet_id":32,"source":"trickbot.pcap","alias":"nDPId-test","flow_id":1,"flow_state":"finished","flow_src_packets_processed":9,"flow_dst_packets_processed":23,"flow_first_seen":1609266107551500,"flow_src_last_pkt_time":1609266109737227,"flow_dst_last_pkt_time":1609266110219915,"flow_idle_time":3285032704,"flow_src_min_l4_payload_len":0,"flow_dst_min_l4_payload_len":0,"flow_src_max_l4_payload_len":928,"flow_dst_max_l4_payload_len":1460,"flow_src_tot_l4_payload_len":1277,"flow_dst_tot_l4_payload_len":27187,"midstream":0,"thread_ts_usec":1609266110219915,"l3_proto":"ip4","src_ip":"10.12.29.101","dst_ip":"82.118.225.196","src_port":61318,"dst_port":7080,"l4_proto":"tcp","flow_datalink":1,"flow_max_packets":3,"data_analysis": {"iat": {"min":6,"avg":156585.2,"max":931328,"stddev":258444.3,"var":66793451520.0,"ent":3.3,"data": [245675,245918,203,81,530,37,931085,931328,2339,2280,480234,19,480300,297566,15,8,7,8,7,8,8,7,7,6,9,297680,227938,227937,482874,14,14]},"pktlen": {"min":54,"avg":944.0,"max":1514,"stddev":662.5,"var":438885.5,"ent":4.5,"data": [66,58,54,403,982,54,54,1412,54,1412,54,1514,1337,54,1514,1514,1514,1514,1514,1514,1514,1514,1514,1514,1514,290,54,1412,54,1514,1514,1208]},"bins": {"c_to_s": [7,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"s_to_c": [3,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,1,0,3,0,0,14,0,0]},"directions": [0,1,0,0,0,1,1,1,0,1,0,1,1,0,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,1,1,1]},"ndpi": {"flow_risk": {"5": {"risk":"Known Proto on Non Std Port","severity":"Medium","risk_score": {"total":260,"client":230,"server":30}},"12": {"risk":"HTTP Numeric IP Address","severity":"Low","risk_score": {"total":500,"client":450,"server":50}},"25": {"risk":"HTTP Suspicious Content","severity":"High","risk_score": {"total":510,"client":355,"server":155}}},"confidence": {"6":"DPI"},"proto":"HTTP","proto_id":"7","encrypted":0,"breed":"Acceptable","category_id":5,"category":"Web"}}
 01264{"flow_event_id":2,"flow_event_name":"end","thread_id":0,"packet_id":74,"source":"trickbot.pcap","alias":"nDPId-test","flow_id":1,"flow_state":"finished","flow_src_packets_processed":28,"flow_dst_packets_processed":46,"flow_first_seen":1609266107551500,"flow_src_last_pkt_time":1609266115947454,"flow_dst_last_pkt_time":1609266115947521,"flow_idle_time":3285032704,"flow_src_min_l4_payload_len":0,"flow_dst_min_l4_payload_len":0,"flow_src_max_l4_payload_len":928,"flow_dst_max_l4_payload_len":1460,"flow_src_tot_l4_payload_len":1277,"flow_dst_tot_l4_payload_len":56713,"midstream":0,"thread_ts_usec":1609266115947521,"l3_proto":"ip4","src_ip":"10.12.29.101","dst_ip":"82.118.225.196","src_port":61318,"dst_port":7080,"l4_proto":"tcp","flow_datalink":1,"flow_max_packets":3,"ndpi": {"flow_risk": {"5": {"risk":"Known Proto on Non Std Port","severity":"Medium","risk_score": {"total":260,"client":230,"server":30}},"12": {"risk":"HTTP Numeric IP Address","severity":"Low","risk_score": {"total":500,"client":450,"server":50}},"25": {"risk":"HTTP Suspicious Content","severity":"High","risk_score": {"total":510,"client":355,"server":155}}},"confidence": {"6":"DPI"},"proto":"HTTP","proto_id":"7","encrypted":0,"breed":"Acceptable","category_id":5,"category":"Web"}}
 00561{"daemon_event_id":3,"daemon_event_name":"shutdown","thread_id":0,"packet_id":74,"source":"trickbot.pcap","alias":"nDPId-test","packets-captured":74,"packets-processed":74,"total-skipped-flows":0,"total-l4-payload-len":57990,"total-not-detected-flows":0,"total-guessed-flows":0,"total-detected-flows":1,"total-detection-updates":1,"total-updates":0,"current-active-flows":0,"total-active-flows":1,"total-idle-flows":1,"total-compressions":0,"total-compression-diff":0,"current-compression-diff":0,"total-events-serialized":11,"global_ts_usec":1609266115947521}
 ~~~~~~~~~~~~~~~~~~~~ SUMMARY ~~~~~~~~~~~~~~~~~~~~
@@ -17,10 +17,10 @@
 ~~ total active/idle flows...: 1/1
 ~~ total timeout flows.......: 0
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-~~ total memory allocated....: 6038026 bytes
-~~ total memory freed........: 6038026 bytes
+~~ total memory allocated....: 6038022 bytes
+~~ total memory freed........: 6038022 bytes
 ~~ total allocations/frees...: 121568/121568
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 ~~ json string min len.......: 493 chars
-~~ json string max len.......: 2067 chars
+~~ json string max len.......: 2065 chars
 ~~ json string avg len.......: 1232 chars