Refactored client distributor C API.

 * Still not perfect, but the code before was not even able to deal with JSON arrays.
   Use common "speaking" function names for all functions in nDPIsrvd.h
 * Provide a more or less generic and easy extendable JSON walk function.
 * Modified C examples to align with the changed C API.
 * c-collectd: Reduced lot's of code duplication by providing mapping tables.
 * nDPId: IAT array requires one slot less (first packet has always an IAT of 0).

Signed-off-by: Toni Uhlig <matzeton@googlemail.com>

1 files changed, 6 insertions, 6 deletions

diff --git a/test/results/ftp.pcap.out b/test/results/ftp.pcap.out
index e979af4e0..ea6b69119 100644
--- a/test/results/ftp.pcap.out
+++ b/test/results/ftp.pcap.out
@@ -5,7 +5,7 @@
 00532{"packet_event_id":2,"packet_event_name":"packet-flow","thread_id":0,"packet_id":2,"source":"ftp.pcap","alias":"nDPId-test","flow_id":1,"flow_packet_id":2,"flow_src_last_pkt_time":1552590234892296,"flow_dst_last_pkt_time":1552590234919708,"flow_idle_time":3285032704,"pkt_oversize":false,"pkt_caplen":74,"pkt_type":2048,"pkt_l3_offset":14,"pkt_l4_offset":34,"pkt_len":74,"pkt_l4_len":40,"thread_ts_usec":1552590234919708,"pkt":"xCwDBkn+EBMx8Tl2CABFAAA8AABAADYG4XRagkZJwKgB1AAVxgZYKsHSoyOX7qASqbA+KAAAAgQFrAQCCAoSZ\/tNO1eYmQEDAw4="}
 00520{"packet_event_id":2,"packet_event_name":"packet-flow","thread_id":0,"packet_id":3,"source":"ftp.pcap","alias":"nDPId-test","flow_id":1,"flow_packet_id":3,"flow_src_last_pkt_time":1552590234919816,"flow_dst_last_pkt_time":1552590234919708,"flow_idle_time":3285032704,"pkt_oversize":false,"pkt_caplen":66,"pkt_type":2048,"pkt_l3_offset":14,"pkt_l4_offset":34,"pkt_len":66,"pkt_l4_len":32,"thread_ts_usec":1552590234919816,"pkt":"EBMx8Tl2xCwDBkn+CABFAAA0AABAAEAGAADAqAHUWoJGScYGABWjI5fuWCrB04AQECxjbgAAAQEICjtXmLQSZ\/tN"}
 01043{"flow_event_id":7,"flow_event_name":"detected","thread_id":0,"packet_id":12,"source":"ftp.pcap","alias":"nDPId-test","flow_id":1,"flow_state":"info","flow_src_packets_processed":6,"flow_dst_packets_processed":6,"flow_first_seen":1552590234892296,"flow_src_last_pkt_time":1552590234976972,"flow_dst_last_pkt_time":1552590235066945,"flow_idle_time":3285032704,"flow_src_min_l4_payload_len":0,"flow_dst_min_l4_payload_len":0,"flow_src_max_l4_payload_len":16,"flow_dst_max_l4_payload_len":34,"flow_src_tot_l4_payload_len":29,"flow_dst_tot_l4_payload_len":77,"midstream":0,"thread_ts_usec":1552590235066945,"l3_proto":"ip4","src_ip":"192.168.1.212","dst_ip":"90.130.70.73","src_port":50694,"dst_port":21,"l4_proto":"tcp","ndpi": {"flow_risk": {"22": {"risk":"Unsafe Protocol","severity":"Low","risk_score": {"total":750,"client":575,"server":175}}},"confidence": {"6":"DPI"},"proto":"FTP_CONTROL","proto_id":"1","encrypted":0,"breed":"Unsafe","category_id":7,"category":"Download","ftp": {"user":"anonymous","password":"NcFTP@","auth_failed":0}}}
-01795{"flow_event_id":5,"flow_event_name":"analyse","thread_id":0,"packet_id":32,"source":"ftp.pcap","alias":"nDPId-test","flow_id":1,"flow_state":"finished","flow_src_packets_processed":18,"flow_dst_packets_processed":14,"flow_first_seen":1552590234892296,"flow_src_last_pkt_time":1552590235175924,"flow_dst_last_pkt_time":1552590235202548,"flow_idle_time":3285032704,"flow_src_min_l4_payload_len":0,"flow_dst_min_l4_payload_len":0,"flow_src_max_l4_payload_len":30,"flow_dst_max_l4_payload_len":241,"flow_src_tot_l4_payload_len":86,"flow_dst_tot_l4_payload_len":532,"midstream":0,"thread_ts_usec":1552590235202548,"l3_proto":"ip4","src_ip":"192.168.1.212","dst_ip":"90.130.70.73","src_port":50694,"dst_port":21,"l4_proto":"tcp","flow_datalink":1,"flow_max_packets":3,"data_analysis": {"iat": {"min":6,"avg":19157.4,"max":90047,"stddev":20644.4,"var":426190272.0,"ent":4.1,"data": [27412,27520,29008,29012,526,27660,315,27401,217,69061,21193,90047,306,27070,21,26780,133,26972,64,26857,6,275,27478,27261,90,29,651,27147,26517,90,26761,0]},"pktlen": {"min":66,"avg":85.9,"max":307,"stddev":42.7,"var":1824.0,"ent":4.9,"data": [78,74,66,86,66,82,66,100,66,79,66,89,66,71,66,100,66,72,81,131,66,66,77,110,66,307,66,96,88,66,71,100]},"bins": {"c_to_s": [18,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"s_to_c": [8,4,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]},"directions": [0,1,0,1,0,0,1,1,0,0,1,1,0,0,1,1,0,0,1,1,0,0,0,1,0,1,0,0,1,0,0,1]},"ndpi": {"flow_risk": {"22": {"risk":"Unsafe Protocol","severity":"Low","risk_score": {"total":750,"client":575,"server":175}}},"confidence": {"6":"DPI"},"proto":"FTP_CONTROL","proto_id":"1","encrypted":0,"breed":"Unsafe","category_id":7,"category":"Download"}}
+01793{"flow_event_id":5,"flow_event_name":"analyse","thread_id":0,"packet_id":32,"source":"ftp.pcap","alias":"nDPId-test","flow_id":1,"flow_state":"finished","flow_src_packets_processed":18,"flow_dst_packets_processed":14,"flow_first_seen":1552590234892296,"flow_src_last_pkt_time":1552590235175924,"flow_dst_last_pkt_time":1552590235202548,"flow_idle_time":3285032704,"flow_src_min_l4_payload_len":0,"flow_dst_min_l4_payload_len":0,"flow_src_max_l4_payload_len":30,"flow_dst_max_l4_payload_len":241,"flow_src_tot_l4_payload_len":86,"flow_dst_tot_l4_payload_len":532,"midstream":0,"thread_ts_usec":1552590235202548,"l3_proto":"ip4","src_ip":"192.168.1.212","dst_ip":"90.130.70.73","src_port":50694,"dst_port":21,"l4_proto":"tcp","flow_datalink":1,"flow_max_packets":3,"data_analysis": {"iat": {"min":6,"avg":19157.4,"max":90047,"stddev":20644.4,"var":426190272.0,"ent":4.1,"data": [27412,27520,29008,29012,526,27660,315,27401,217,69061,21193,90047,306,27070,21,26780,133,26972,64,26857,6,275,27478,27261,90,29,651,27147,26517,90,26761]},"pktlen": {"min":66,"avg":85.9,"max":307,"stddev":42.7,"var":1824.0,"ent":4.9,"data": [78,74,66,86,66,82,66,100,66,79,66,89,66,71,66,100,66,72,81,131,66,66,77,110,66,307,66,96,88,66,71,100]},"bins": {"c_to_s": [18,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"s_to_c": [8,4,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]},"directions": [0,1,0,1,0,0,1,1,0,0,1,1,0,0,1,1,0,0,1,1,0,0,0,1,0,1,0,0,1,0,0,1]},"ndpi": {"flow_risk": {"22": {"risk":"Unsafe Protocol","severity":"Low","risk_score": {"total":750,"client":575,"server":175}}},"confidence": {"6":"DPI"},"proto":"FTP_CONTROL","proto_id":"1","encrypted":0,"breed":"Unsafe","category_id":7,"category":"Download"}}
 00751{"flow_event_id":1,"flow_event_name":"new","thread_id":0,"packet_id":37,"source":"ftp.pcap","alias":"nDPId-test","flow_id":2,"flow_state":"info","flow_src_packets_processed":1,"flow_dst_packets_processed":0,"flow_first_seen":1552590236580045,"flow_src_last_pkt_time":1552590236580045,"flow_dst_last_pkt_time":1552590236580045,"flow_idle_time":3285032704,"flow_src_min_l4_payload_len":0,"flow_dst_min_l4_payload_len":0,"flow_src_max_l4_payload_len":0,"flow_dst_max_l4_payload_len":0,"flow_src_tot_l4_payload_len":0,"flow_dst_tot_l4_payload_len":0,"midstream":0,"thread_ts_usec":1552590236580045,"l3_proto":"ip4","src_ip":"192.168.1.212","dst_ip":"90.130.70.73","src_port":50695,"dst_port":25685,"l4_proto":"tcp","flow_datalink":1,"flow_max_packets":3}
 00538{"packet_event_id":2,"packet_event_name":"packet-flow","thread_id":0,"packet_id":37,"source":"ftp.pcap","alias":"nDPId-test","flow_id":2,"flow_packet_id":1,"flow_src_last_pkt_time":1552590236580045,"flow_dst_last_pkt_time":1552590236580045,"flow_idle_time":3285032704,"pkt_oversize":false,"pkt_caplen":78,"pkt_type":2048,"pkt_l3_offset":14,"pkt_l4_offset":34,"pkt_len":78,"pkt_l4_len":44,"thread_ts_usec":1552590236580045,"pkt":"EBMx8Tl2xCwDBkn+CABFAABAAABAAEAGAADAqAHUWoJGScYHZFXuwKKMAAAAALAC\/\/9jegAAAgQFtAEDAwUBAQgKO1efIQAAAAAEAgAA"}
 00533{"packet_event_id":2,"packet_event_name":"packet-flow","thread_id":0,"packet_id":38,"source":"ftp.pcap","alias":"nDPId-test","flow_id":2,"flow_packet_id":2,"flow_src_last_pkt_time":1552590236580045,"flow_dst_last_pkt_time":1552590236608252,"flow_idle_time":3285032704,"pkt_oversize":false,"pkt_caplen":74,"pkt_type":2048,"pkt_l3_offset":14,"pkt_l4_offset":34,"pkt_len":74,"pkt_l4_len":40,"thread_ts_usec":1552590236608252,"pkt":"xCwDBkn+EBMx8Tl2CABFAAA8AABAADYG4XRagkZJwKgB1GRVxgdmK2Nw7sCijaASqbDL3QAAAgQFrAQCCAoSZ\/zzO1efIQEDAw4="}
@@ -15,7 +15,7 @@
 00538{"packet_event_id":2,"packet_event_name":"packet-flow","thread_id":0,"packet_id":66,"source":"ftp.pcap","alias":"nDPId-test","flow_id":3,"flow_packet_id":1,"flow_src_last_pkt_time":1552590241545143,"flow_dst_last_pkt_time":1552590241545143,"flow_idle_time":3285032704,"pkt_oversize":false,"pkt_caplen":78,"pkt_type":2048,"pkt_l3_offset":14,"pkt_l4_offset":34,"pkt_len":78,"pkt_l4_len":44,"thread_ts_usec":1552590241545143,"pkt":"EBMx8Tl2xCwDBkn+CABFAABAAABAAEAGAADAqAHUWoJGScYIX8sNBxpOAAAAALAC\/\/9jegAAAgQFtAEDAwUBAQgKO1eyYgAAAAAEAgAA"}
 00533{"packet_event_id":2,"packet_event_name":"packet-flow","thread_id":0,"packet_id":67,"source":"ftp.pcap","alias":"nDPId-test","flow_id":3,"flow_packet_id":2,"flow_src_last_pkt_time":1552590241545143,"flow_dst_last_pkt_time":1552590241573913,"flow_idle_time":3285032704,"pkt_oversize":false,"pkt_caplen":74,"pkt_type":2048,"pkt_l3_offset":14,"pkt_l4_offset":34,"pkt_len":74,"pkt_l4_len":40,"thread_ts_usec":1552590241573913,"pkt":"xCwDBkn+EBMx8Tl2CABFAAA8AABAADYG4XRagkZJwKgB1F\/LxggMTnkwDQcaT6ASqbBmYgAAAgQFrAQCCAoSaAHMO1eyYgEDAw4="}
 00520{"packet_event_id":2,"packet_event_name":"packet-flow","thread_id":0,"packet_id":68,"source":"ftp.pcap","alias":"nDPId-test","flow_id":3,"flow_packet_id":3,"flow_src_last_pkt_time":1552590241573957,"flow_dst_last_pkt_time":1552590241573913,"flow_idle_time":3285032704,"pkt_oversize":false,"pkt_caplen":66,"pkt_type":2048,"pkt_l3_offset":14,"pkt_l4_offset":34,"pkt_len":66,"pkt_l4_len":32,"thread_ts_usec":1552590241573957,"pkt":"EBMx8Tl2xCwDBkn+CABFAAA0AABAAEAGAADAqAHUWoJGScYIX8sNBxpPDE55MYAQECxjbgAAAQEICjtXsn0SaAHM"}
-01550{"flow_event_id":5,"flow_event_name":"analyse","thread_id":0,"packet_id":100,"source":"ftp.pcap","alias":"nDPId-test","flow_id":3,"flow_state":"info","flow_src_packets_processed":13,"flow_dst_packets_processed":19,"flow_first_seen":1552590241545143,"flow_src_last_pkt_time":1552590241637688,"flow_dst_last_pkt_time":1552590241639633,"flow_idle_time":3285032704,"flow_src_min_l4_payload_len":0,"flow_dst_min_l4_payload_len":0,"flow_src_max_l4_payload_len":0,"flow_dst_max_l4_payload_len":1440,"flow_src_tot_l4_payload_len":0,"flow_dst_tot_l4_payload_len":24480,"midstream":0,"thread_ts_usec":1552590241639633,"l3_proto":"ip4","src_ip":"192.168.1.212","dst_ip":"90.130.70.73","src_port":50696,"dst_port":24523,"l4_proto":"tcp","flow_datalink":1,"flow_max_packets":3,"data_analysis": {"iat": {"min":2,"avg":6033.4,"max":29579,"stddev":11108.9,"var":123407192.0,"ent":3.1,"data": [28770,28814,29579,29566,281,284,597,608,340,458,790,363,375,64,327,2,379,43,300,27513,27767,195,211,1702,115,4,1805,1866,1903,218,1796,0]},"pktlen": {"min":66,"avg":832.0,"max":1506,"stddev":717.5,"var":514855.0,"ent":4.3,"data": [78,74,66,1506,78,1506,66,1506,66,1506,1506,66,1506,66,1506,1506,1506,66,66,1506,1506,66,1506,66,1506,1506,66,66,1506,66,1506,1506]},"bins": {"c_to_s": [13,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"s_to_c": [2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,17,0,0]},"directions": [0,1,0,1,0,1,0,1,0,1,1,0,1,0,1,1,1,0,0,1,1,0,1,0,1,1,1,0,1,0,1,1]}}
+01548{"flow_event_id":5,"flow_event_name":"analyse","thread_id":0,"packet_id":100,"source":"ftp.pcap","alias":"nDPId-test","flow_id":3,"flow_state":"info","flow_src_packets_processed":13,"flow_dst_packets_processed":19,"flow_first_seen":1552590241545143,"flow_src_last_pkt_time":1552590241637688,"flow_dst_last_pkt_time":1552590241639633,"flow_idle_time":3285032704,"flow_src_min_l4_payload_len":0,"flow_dst_min_l4_payload_len":0,"flow_src_max_l4_payload_len":0,"flow_dst_max_l4_payload_len":1440,"flow_src_tot_l4_payload_len":0,"flow_dst_tot_l4_payload_len":24480,"midstream":0,"thread_ts_usec":1552590241639633,"l3_proto":"ip4","src_ip":"192.168.1.212","dst_ip":"90.130.70.73","src_port":50696,"dst_port":24523,"l4_proto":"tcp","flow_datalink":1,"flow_max_packets":3,"data_analysis": {"iat": {"min":2,"avg":6033.4,"max":29579,"stddev":11108.9,"var":123407192.0,"ent":3.1,"data": [28770,28814,29579,29566,281,284,597,608,340,458,790,363,375,64,327,2,379,43,300,27513,27767,195,211,1702,115,4,1805,1866,1903,218,1796]},"pktlen": {"min":66,"avg":832.0,"max":1506,"stddev":717.5,"var":514855.0,"ent":4.3,"data": [78,74,66,1506,78,1506,66,1506,66,1506,1506,66,1506,66,1506,1506,1506,66,66,1506,1506,66,1506,66,1506,1506,66,66,1506,66,1506,1506]},"bins": {"c_to_s": [13,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"s_to_c": [2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,17,0,0]},"directions": [0,1,0,1,0,1,0,1,0,1,1,0,1,0,1,1,1,0,0,1,1,0,1,0,1,1,1,0,1,0,1,1]}}
 00806{"flow_event_id":9,"flow_event_name":"not-detected","thread_id":0,"packet_id":100,"source":"ftp.pcap","alias":"nDPId-test","flow_id":3,"flow_state":"info","flow_src_packets_processed":13,"flow_dst_packets_processed":19,"flow_first_seen":1552590241545143,"flow_src_last_pkt_time":1552590241637688,"flow_dst_last_pkt_time":1552590241639633,"flow_idle_time":3285032704,"flow_src_min_l4_payload_len":0,"flow_dst_min_l4_payload_len":0,"flow_src_max_l4_payload_len":0,"flow_dst_max_l4_payload_len":1440,"flow_src_tot_l4_payload_len":0,"flow_dst_tot_l4_payload_len":24480,"midstream":0,"thread_ts_usec":1552590241639633,"l3_proto":"ip4","src_ip":"192.168.1.212","dst_ip":"90.130.70.73","src_port":50696,"dst_port":24523,"l4_proto":"tcp","ndpi": {"proto":"Unknown","proto_id":"0","encrypted":0,"breed":"Unrated"}}
 00845{"flow_event_id":2,"flow_event_name":"end","thread_id":0,"packet_id":1192,"source":"ftp.pcap","alias":"nDPId-test","flow_id":3,"flow_state":"finished","flow_src_packets_processed":380,"flow_dst_packets_processed":735,"flow_first_seen":1552590241545143,"flow_src_last_pkt_time":1552590241851108,"flow_dst_last_pkt_time":1552590241878454,"flow_idle_time":3285032704,"flow_src_min_l4_payload_len":0,"flow_dst_min_l4_payload_len":0,"flow_src_max_l4_payload_len":0,"flow_dst_max_l4_payload_len":1440,"flow_src_tot_l4_payload_len":0,"flow_dst_tot_l4_payload_len":1048576,"midstream":0,"thread_ts_usec":1552590243371057,"l3_proto":"ip4","src_ip":"192.168.1.212","dst_ip":"90.130.70.73","src_port":50696,"dst_port":24523,"l4_proto":"tcp","flow_datalink":1,"flow_max_packets":3,"ndpi": {"proto":"Unknown","proto_id":"0","encrypted":0,"breed":"Unrated"}}
 01024{"flow_event_id":2,"flow_event_name":"end","thread_id":0,"packet_id":1192,"source":"ftp.pcap","alias":"nDPId-test","flow_id":1,"flow_state":"finished","flow_src_packets_processed":41,"flow_dst_packets_processed":27,"flow_first_seen":1552590234892296,"flow_src_last_pkt_time":1552590243340268,"flow_dst_last_pkt_time":1552590243371057,"flow_idle_time":3285032704,"flow_src_min_l4_payload_len":0,"flow_dst_min_l4_payload_len":0,"flow_src_max_l4_payload_len":30,"flow_dst_max_l4_payload_len":241,"flow_src_tot_l4_payload_len":174,"flow_dst_tot_l4_payload_len":889,"midstream":0,"thread_ts_usec":1552590243371057,"l3_proto":"ip4","src_ip":"192.168.1.212","dst_ip":"90.130.70.73","src_port":50694,"dst_port":21,"l4_proto":"tcp","flow_datalink":1,"flow_max_packets":3,"ndpi": {"flow_risk": {"22": {"risk":"Unsafe Protocol","severity":"Low","risk_score": {"total":750,"client":575,"server":175}}},"confidence": {"6":"DPI"},"proto":"FTP_CONTROL","proto_id":"1","encrypted":0,"breed":"Unsafe","category_id":7,"category":"Download"}}
@@ -29,10 +29,10 @@
 ~~ total active/idle flows...: 3/3
 ~~ total timeout flows.......: 0
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-~~ total memory allocated....: 6079644 bytes
-~~ total memory freed........: 6079644 bytes
+~~ total memory allocated....: 6079632 bytes
+~~ total memory freed........: 6079632 bytes
 ~~ total allocations/frees...: 122703/122703
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 ~~ json string min len.......: 488 chars
-~~ json string max len.......: 1800 chars
-~~ json string avg len.......: 1143 chars
+~~ json string max len.......: 1798 chars
+~~ json string avg len.......: 1142 chars