Refactored client distributor C API.

 * Still not perfect, but the code before was not even able to deal with JSON arrays.
   Use common "speaking" function names for all functions in nDPIsrvd.h
 * Provide a more or less generic and easy extendable JSON walk function.
 * Modified C examples to align with the changed C API.
 * c-collectd: Reduced lot's of code duplication by providing mapping tables.
 * nDPId: IAT array requires one slot less (first packet has always an IAT of 0).

Signed-off-by: Toni Uhlig <matzeton@googlemail.com>

1 files changed, 6 insertions, 6 deletions

diff --git a/test/results/fix2.pcap.out b/test/results/fix2.pcap.out
index 7b13ba278..9863e4d62 100644
--- a/test/results/fix2.pcap.out
+++ b/test/results/fix2.pcap.out
@@ -10,8 +10,8 @@
 00517{"packet_event_id":2,"packet_event_name":"packet-flow","thread_id":0,"packet_id":9,"source":"fix2.pcap","alias":"nDPId-test","flow_id":2,"flow_packet_id":2,"flow_src_last_pkt_time":1614758889589020,"flow_dst_last_pkt_time":1614758889589588,"flow_idle_time":3285032704,"pkt_oversize":false,"pkt_caplen":62,"pkt_type":2048,"pkt_l3_offset":14,"pkt_l4_offset":34,"pkt_len":62,"pkt_l4_len":28,"thread_ts_usec":1614758889589588,"pkt":"WgXZu6TVApXG95WRCABFAAAweT8AAIAGrLMKZgAJCmUAAgQAiJMt1EWWLdRCK3ASgAF\/OwAAAgQFtAMDAQA="}
 00513{"packet_event_id":2,"packet_event_name":"packet-flow","thread_id":0,"packet_id":10,"source":"fix2.pcap","alias":"nDPId-test","flow_id":2,"flow_packet_id":3,"flow_src_last_pkt_time":1614758889589590,"flow_dst_last_pkt_time":1614758889589588,"flow_idle_time":3285032704,"pkt_oversize":false,"pkt_caplen":60,"pkt_type":2048,"pkt_l3_offset":14,"pkt_l4_offset":34,"pkt_len":60,"pkt_l4_len":20,"thread_ts_usec":1614758889589590,"pkt":"5kBKB+riApXG95NLCABFAAAoeUAAAIAGAAAKZQACCmYACYiTBAAt1EIrLdRFl1AQgAEU8AAAAAAAAAAA"}
 00844{"flow_event_id":7,"flow_event_name":"detected","thread_id":0,"packet_id":11,"source":"fix2.pcap","alias":"nDPId-test","flow_id":2,"flow_state":"info","flow_src_packets_processed":3,"flow_dst_packets_processed":1,"flow_first_seen":1614758889589020,"flow_src_last_pkt_time":1614758889589592,"flow_dst_last_pkt_time":1614758889589588,"flow_idle_time":3285032704,"flow_src_min_l4_payload_len":0,"flow_dst_min_l4_payload_len":0,"flow_src_max_l4_payload_len":85,"flow_dst_max_l4_payload_len":0,"flow_src_tot_l4_payload_len":85,"flow_dst_tot_l4_payload_len":0,"midstream":0,"thread_ts_usec":1614758889589592,"l3_proto":"ip4","src_ip":"10.101.0.2","dst_ip":"10.102.0.9","src_port":34963,"dst_port":1024,"l4_proto":"tcp","ndpi": {"confidence": {"6":"DPI"},"proto":"FIX","proto_id":"230","encrypted":0,"breed":"Safe","category_id":16,"category":"RPC"}}
-01593{"flow_event_id":5,"flow_event_name":"analyse","thread_id":0,"packet_id":56,"source":"fix2.pcap","alias":"nDPId-test","flow_id":1,"flow_state":"finished","flow_src_packets_processed":14,"flow_dst_packets_processed":18,"flow_first_seen":1614758889588862,"flow_src_last_pkt_time":1614758889589960,"flow_dst_last_pkt_time":1614758889589962,"flow_idle_time":3285032704,"flow_src_min_l4_payload_len":0,"flow_dst_min_l4_payload_len":0,"flow_src_max_l4_payload_len":106,"flow_dst_max_l4_payload_len":120,"flow_src_tot_l4_payload_len":669,"flow_dst_tot_l4_payload_len":911,"midstream":0,"thread_ts_usec":1614758889589962,"l3_proto":"ip4","src_ip":"10.101.0.2","dst_ip":"10.102.0.2","src_port":34962,"dst_port":1024,"l4_proto":"tcp","flow_datalink":1,"flow_max_packets":3,"data_analysis": {"iat": {"min":2,"avg":73.3,"max":652,"stddev":161.3,"var":26006.9,"ent":3.1,"data": [641,652,12,92,71,9,33,29,203,208,31,32,5,2,23,28,2,2,8,8,11,13,25,23,5,4,9,5,7,5,0,0]},"pktlen": {"min":60,"avg":106.6,"max":174,"stddev":46.7,"var":2179.9,"ent":4.9,"data": [62,62,60,139,62,60,147,144,60,152,144,152,146,60,60,147,60,60,60,152,60,174,157,174,60,60,60,60,157,147,160,152]},"bins": {"c_to_s": [7,0,4,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"s_to_c": [9,0,3,6,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]},"directions": [0,1,0,0,1,0,1,0,1,1,0,1,0,1,1,1,0,1,0,1,0,1,0,1,0,1,1,1,0,1,0,1]},"ndpi": {"confidence": {"6":"DPI"},"proto":"FIX","proto_id":"230","encrypted":0,"breed":"Safe","category_id":16,"category":"RPC"}}
-01599{"flow_event_id":5,"flow_event_name":"analyse","thread_id":0,"packet_id":79,"source":"fix2.pcap","alias":"nDPId-test","flow_id":2,"flow_state":"finished","flow_src_packets_processed":14,"flow_dst_packets_processed":18,"flow_first_seen":1614758889589020,"flow_src_last_pkt_time":1614758889590049,"flow_dst_last_pkt_time":1614758889590048,"flow_idle_time":3285032704,"flow_src_min_l4_payload_len":0,"flow_dst_min_l4_payload_len":0,"flow_src_max_l4_payload_len":106,"flow_dst_max_l4_payload_len":120,"flow_src_tot_l4_payload_len":762,"flow_dst_tot_l4_payload_len":801,"midstream":0,"thread_ts_usec":1614758889590049,"l3_proto":"ip4","src_ip":"10.101.0.2","dst_ip":"10.102.0.9","src_port":34963,"dst_port":1024,"l4_proto":"tcp","flow_datalink":1,"flow_max_packets":3,"data_analysis": {"iat": {"min":1,"avg":70.9,"max":570,"stddev":141.3,"var":19970.8,"ent":3.3,"data": [568,570,2,146,145,106,1,105,2,16,6,26,48,7,14,19,2,2,18,19,48,49,27,12,37,4,6,27,25,0,0,0]},"pktlen": {"min":60,"avg":106.0,"max":174,"stddev":46.1,"var":2122.5,"ent":4.9,"data": [62,62,60,139,147,144,152,62,60,144,60,60,152,146,60,147,60,152,60,174,157,147,160,60,60,60,160,162,144,60,60,60]},"bins": {"c_to_s": [6,0,5,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"s_to_c": [10,0,3,5,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]},"directions": [0,1,0,0,1,0,1,1,0,0,1,1,1,0,1,1,0,1,0,1,0,1,0,1,1,1,0,1,0,1,1,0]},"ndpi": {"confidence": {"6":"DPI"},"proto":"FIX","proto_id":"230","encrypted":0,"breed":"Safe","category_id":16,"category":"RPC"}}
+01589{"flow_event_id":5,"flow_event_name":"analyse","thread_id":0,"packet_id":56,"source":"fix2.pcap","alias":"nDPId-test","flow_id":1,"flow_state":"finished","flow_src_packets_processed":14,"flow_dst_packets_processed":18,"flow_first_seen":1614758889588862,"flow_src_last_pkt_time":1614758889589960,"flow_dst_last_pkt_time":1614758889589962,"flow_idle_time":3285032704,"flow_src_min_l4_payload_len":0,"flow_dst_min_l4_payload_len":0,"flow_src_max_l4_payload_len":106,"flow_dst_max_l4_payload_len":120,"flow_src_tot_l4_payload_len":669,"flow_dst_tot_l4_payload_len":911,"midstream":0,"thread_ts_usec":1614758889589962,"l3_proto":"ip4","src_ip":"10.101.0.2","dst_ip":"10.102.0.2","src_port":34962,"dst_port":1024,"l4_proto":"tcp","flow_datalink":1,"flow_max_packets":3,"data_analysis": {"iat": {"min":2,"avg":73.3,"max":652,"stddev":161.3,"var":26006.9,"ent":3.1,"data": [641,652,12,92,71,9,33,29,203,208,31,32,5,2,23,28,2,2,8,8,11,13,25,23,5,4,9,5,7,5]},"pktlen": {"min":60,"avg":106.6,"max":174,"stddev":46.7,"var":2179.9,"ent":4.9,"data": [62,62,60,139,62,60,147,144,60,152,144,152,146,60,60,147,60,60,60,152,60,174,157,174,60,60,60,60,157,147,160,152]},"bins": {"c_to_s": [7,0,4,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"s_to_c": [9,0,3,6,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]},"directions": [0,1,0,0,1,0,1,0,1,1,0,1,0,1,1,1,0,1,0,1,0,1,0,1,0,1,1,1,0,1,0,1]},"ndpi": {"confidence": {"6":"DPI"},"proto":"FIX","proto_id":"230","encrypted":0,"breed":"Safe","category_id":16,"category":"RPC"}}
+01593{"flow_event_id":5,"flow_event_name":"analyse","thread_id":0,"packet_id":79,"source":"fix2.pcap","alias":"nDPId-test","flow_id":2,"flow_state":"finished","flow_src_packets_processed":14,"flow_dst_packets_processed":18,"flow_first_seen":1614758889589020,"flow_src_last_pkt_time":1614758889590049,"flow_dst_last_pkt_time":1614758889590048,"flow_idle_time":3285032704,"flow_src_min_l4_payload_len":0,"flow_dst_min_l4_payload_len":0,"flow_src_max_l4_payload_len":106,"flow_dst_max_l4_payload_len":120,"flow_src_tot_l4_payload_len":762,"flow_dst_tot_l4_payload_len":801,"midstream":0,"thread_ts_usec":1614758889590049,"l3_proto":"ip4","src_ip":"10.101.0.2","dst_ip":"10.102.0.9","src_port":34963,"dst_port":1024,"l4_proto":"tcp","flow_datalink":1,"flow_max_packets":3,"data_analysis": {"iat": {"min":1,"avg":70.9,"max":570,"stddev":141.3,"var":19970.8,"ent":3.3,"data": [568,570,2,146,145,106,1,105,2,16,6,26,48,7,14,19,2,2,18,19,48,49,27,12,37,4,6,27,25]},"pktlen": {"min":60,"avg":106.0,"max":174,"stddev":46.1,"var":2122.5,"ent":4.9,"data": [62,62,60,139,147,144,152,62,60,144,60,60,152,146,60,147,60,152,60,174,157,147,160,60,60,60,160,162,144,60,60,60]},"bins": {"c_to_s": [6,0,5,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"s_to_c": [10,0,3,5,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]},"directions": [0,1,0,0,1,0,1,1,0,0,1,1,1,0,1,1,0,1,0,1,0,1,0,1,1,1,0,1,0,1,1,0]},"ndpi": {"confidence": {"6":"DPI"},"proto":"FIX","proto_id":"230","encrypted":0,"breed":"Safe","category_id":16,"category":"RPC"}}
 00899{"flow_event_id":2,"flow_event_name":"end","thread_id":0,"packet_id":3049,"source":"fix2.pcap","alias":"nDPId-test","flow_id":1,"flow_state":"finished","flow_src_packets_processed":683,"flow_dst_packets_processed":1304,"flow_first_seen":1614758889588862,"flow_src_last_pkt_time":1614758889595345,"flow_dst_last_pkt_time":1614758889595344,"flow_idle_time":3285032704,"flow_src_min_l4_payload_len":0,"flow_dst_min_l4_payload_len":0,"flow_src_max_l4_payload_len":106,"flow_dst_max_l4_payload_len":120,"flow_src_tot_l4_payload_len":13395,"flow_dst_tot_l4_payload_len":26148,"midstream":0,"thread_ts_usec":1614758889595345,"l3_proto":"ip4","src_ip":"10.101.0.2","dst_ip":"10.102.0.2","src_port":34962,"dst_port":1024,"l4_proto":"tcp","flow_datalink":1,"flow_max_packets":3,"ndpi": {"confidence": {"6":"DPI"},"proto":"FIX","proto_id":"230","encrypted":0,"breed":"Safe","category_id":16,"category":"RPC"}}
 00898{"flow_event_id":2,"flow_event_name":"end","thread_id":0,"packet_id":3049,"source":"fix2.pcap","alias":"nDPId-test","flow_id":2,"flow_state":"finished","flow_src_packets_processed":411,"flow_dst_packets_processed":648,"flow_first_seen":1614758889589020,"flow_src_last_pkt_time":1614758889595307,"flow_dst_last_pkt_time":1614758889595305,"flow_idle_time":3285032704,"flow_src_min_l4_payload_len":0,"flow_dst_min_l4_payload_len":0,"flow_src_max_l4_payload_len":106,"flow_dst_max_l4_payload_len":120,"flow_src_tot_l4_payload_len":10864,"flow_dst_tot_l4_payload_len":17549,"midstream":0,"thread_ts_usec":1614758889595345,"l3_proto":"ip4","src_ip":"10.101.0.2","dst_ip":"10.102.0.9","src_port":34963,"dst_port":1024,"l4_proto":"tcp","flow_datalink":1,"flow_max_packets":3,"ndpi": {"confidence": {"6":"DPI"},"proto":"FIX","proto_id":"230","encrypted":0,"breed":"Safe","category_id":16,"category":"RPC"}}
 00563{"daemon_event_id":3,"daemon_event_name":"shutdown","thread_id":0,"packet_id":3049,"source":"fix2.pcap","alias":"nDPId-test","packets-captured":3049,"packets-processed":3046,"total-skipped-flows":0,"total-l4-payload-len":67956,"total-not-detected-flows":0,"total-guessed-flows":0,"total-detected-flows":2,"total-detection-updates":0,"total-updates":0,"current-active-flows":0,"total-active-flows":2,"total-idle-flows":2,"total-compressions":0,"total-compression-diff":0,"current-compression-diff":0,"total-events-serialized":17,"global_ts_usec":1614758889595345}
@@ -23,10 +23,10 @@
 ~~ total active/idle flows...: 2/2
 ~~ total timeout flows.......: 0
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-~~ total memory allocated....: 6129703 bytes
-~~ total memory freed........: 6129703 bytes
+~~ total memory allocated....: 6129695 bytes
+~~ total memory freed........: 6129695 bytes
 ~~ total allocations/frees...: 124545/124545
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 ~~ json string min len.......: 489 chars
-~~ json string max len.......: 1604 chars
-~~ json string avg len.......: 1034 chars
+~~ json string max len.......: 1598 chars
+~~ json string avg len.......: 1031 chars