Refactored client distributor C API.

 * Still not perfect, but the code before was not even able to deal with JSON arrays.
   Use common "speaking" function names for all functions in nDPIsrvd.h
 * Provide a more or less generic and easy extendable JSON walk function.
 * Modified C examples to align with the changed C API.
 * c-collectd: Reduced lot's of code duplication by providing mapping tables.
 * nDPId: IAT array requires one slot less (first packet has always an IAT of 0).

Signed-off-by: Toni Uhlig <matzeton@googlemail.com>

1 files changed, 3 insertions, 3 deletions

diff --git a/test/results/tls_long_cert.pcap.out b/test/results/tls_long_cert.pcap.out
index 8060373e3..ca2a8bd31 100644
--- a/test/results/tls_long_cert.pcap.out
+++ b/test/results/tls_long_cert.pcap.out
@@ -7,7 +7,7 @@
 01108{"flow_event_id":7,"flow_event_name":"detected","thread_id":0,"packet_id":4,"source":"tls_long_cert.pcap","alias":"nDPId-test","flow_id":1,"flow_state":"info","flow_src_packets_processed":3,"flow_dst_packets_processed":1,"flow_first_seen":1553619078033240,"flow_src_last_pkt_time":1553619078058827,"flow_dst_last_pkt_time":1553619078058439,"flow_idle_time":3285032704,"flow_src_min_l4_payload_len":0,"flow_dst_min_l4_payload_len":0,"flow_src_max_l4_payload_len":517,"flow_dst_max_l4_payload_len":0,"flow_src_tot_l4_payload_len":517,"flow_dst_tot_l4_payload_len":0,"midstream":0,"thread_ts_usec":1553619078058827,"l3_proto":"ip4","src_ip":"192.168.2.126","dst_ip":"104.111.215.93","src_port":60174,"dst_port":443,"l4_proto":"tcp","ndpi": {"confidence": {"6":"DPI"},"proto":"TLS","proto_id":"91","encrypted":1,"breed":"Safe","category_id":5,"category":"Web","hostname":"www.repubblica.it","tls": {"version":"TLSv1.2","ja3":"66918128f1b9b03303d77c6f2eefd128","ja3s":"","unsafe_cipher":0,"cipher":"TLS_NULL_WITH_NULL_NULL","alpn":"h2,http\/1.1","tls_supported_versions":"GREASE,TLSv1.3,TLSv1.2,TLSv1.1,TLSv1"}}}
 01168{"flow_event_id":8,"flow_event_name":"detection-update","thread_id":0,"packet_id":6,"source":"tls_long_cert.pcap","alias":"nDPId-test","flow_id":1,"flow_state":"info","flow_src_packets_processed":3,"flow_dst_packets_processed":3,"flow_first_seen":1553619078033240,"flow_src_last_pkt_time":1553619078058827,"flow_dst_last_pkt_time":1553619078091883,"flow_idle_time":3285032704,"flow_src_min_l4_payload_len":0,"flow_dst_min_l4_payload_len":0,"flow_src_max_l4_payload_len":517,"flow_dst_max_l4_payload_len":1448,"flow_src_tot_l4_payload_len":517,"flow_dst_tot_l4_payload_len":1448,"midstream":0,"thread_ts_usec":1553619078091883,"l3_proto":"ip4","src_ip":"192.168.2.126","dst_ip":"104.111.215.93","src_port":60174,"dst_port":443,"l4_proto":"tcp","ndpi": {"confidence": {"6":"DPI"},"proto":"TLS","proto_id":"91","encrypted":1,"breed":"Safe","category_id":5,"category":"Web","hostname":"www.repubblica.it","tls": {"version":"TLSv1.2","ja3":"66918128f1b9b03303d77c6f2eefd128","ja3s":"35af4c8cd9495354f7d701ce8ad7fd2d","unsafe_cipher":0,"cipher":"TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384","alpn":"h2,http\/1.1","tls_supported_versions":"GREASE,TLSv1.3,TLSv1.2,TLSv1.1,TLSv1"}}}
 02634{"flow_event_id":8,"flow_event_name":"detection-update","thread_id":0,"packet_id":9,"source":"tls_long_cert.pcap","alias":"nDPId-test","flow_id":1,"flow_state":"info","flow_src_packets_processed":4,"flow_dst_packets_processed":5,"flow_first_seen":1553619078033240,"flow_src_last_pkt_time":1553619078093048,"flow_dst_last_pkt_time":1553619078093749,"flow_idle_time":3285032704,"flow_src_min_l4_payload_len":0,"flow_dst_min_l4_payload_len":0,"flow_src_max_l4_payload_len":517,"flow_dst_max_l4_payload_len":1448,"flow_src_tot_l4_payload_len":517,"flow_dst_tot_l4_payload_len":4096,"midstream":0,"thread_ts_usec":1553619078093749,"l3_proto":"ip4","src_ip":"192.168.2.126","dst_ip":"104.111.215.93","src_port":60174,"dst_port":443,"l4_proto":"tcp","ndpi": {"confidence": {"6":"DPI"},"proto":"TLS","proto_id":"91","encrypted":1,"breed":"Safe","category_id":5,"category":"Web","hostname":"www.repubblica.it","tls": {"version":"TLSv1.2","server_names":"www.repstatic.it,repstatic.it,amp-video.lastampa.it,www.repubblica.it,amp-video.deejay.it,amp-video.d.repubblica.it,www.gelestatic.it,oasjs.kataweb.it,video.d.repubblica.it,www.test.capital.it,napoli.repubblica.it,video.ilsecoloxix.it,genova.repubblica.it,cdn.gelestatic.it,video.gelocal.it,media.deejay.it,media.m2o.it,amp-video.espresso.repubblica.it,download.gelocal.it,amp-video.m2o.it,bologna.repubblica.it,torino.repubblica.it,scripts.kataweb.it,palermo.repubblica.it,roma.repubblica.it,video.xl.repubblica.it,amp-video.gelocal.it,video.espresso.repubblica.it,www.capital.it,video.limesonline.com,media.capital.it,syndication-vod-pro.akamai.media.kataweb.it,test.capital.it,video.deejay.it,video.repubblica.it,milano.repubblica.it,video.lanuovasardegna.it,video.m2o.it,parma.repubblica.it,video.3nz.it,syndication-vod-hds.akamai.media.kataweb.it,amp-video.repubblica.it,video.lastampa.it,webfragments.repubblica.it,amp-video.xl.repubblica.it,amp-video.limesonline.com,media.kataweb.it,bari.repubblica.it,syndication-vod-hls.akamai.media.kataweb.it,amp-video.3nz.it,syndication3rd-vod-pro.akamai.media.kataweb.it,firenze.repubblica.it,amp-video.ilsecoloxix.it,amp-video.lanuovasardegna.it,cdn.flv.kataweb.it","ja3":"66918128f1b9b03303d77c6f2eefd128","ja3s":"35af4c8cd9495354f7d701ce8ad7fd2d","unsafe_cipher":0,"cipher":"TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384","issuerDN":"C=US, O=DigiCert Inc, OU=www.digicert.com, CN=GeoTrust RSA CA 2018","subjectDN":"C=IT, ST=Roma, L=Roma, O=GEDI Digital S.r.l., CN=www.repstatic.it","alpn":"h2,http\/1.1","tls_supported_versions":"GREASE,TLSv1.3,TLSv1.2,TLSv1.1,TLSv1","fingerprint":"0C:9F:21:DB:65:A1:BE:EB:D8:89:38:D3:FF:7A:D9:02:8B:F1:60:A1"}}}
-01690{"flow_event_id":5,"flow_event_name":"analyse","thread_id":0,"packet_id":32,"source":"tls_long_cert.pcap","alias":"nDPId-test","flow_id":1,"flow_state":"finished","flow_src_packets_processed":16,"flow_dst_packets_processed":16,"flow_first_seen":1553619078033240,"flow_src_last_pkt_time":1553619078157096,"flow_dst_last_pkt_time":1553619078157742,"flow_idle_time":3285032704,"flow_src_min_l4_payload_len":0,"flow_dst_min_l4_payload_len":0,"flow_src_max_l4_payload_len":836,"flow_dst_max_l4_payload_len":1448,"flow_src_tot_l4_payload_len":1610,"flow_dst_tot_l4_payload_len":13760,"midstream":0,"thread_ts_usec":1553619078157742,"l3_proto":"ip4","src_ip":"192.168.2.126","dst_ip":"104.111.215.93","src_port":60174,"dst_port":443,"l4_proto":"tcp","flow_datalink":1,"flow_max_packets":3,"data_analysis": {"iat": {"min":1,"avg":8011.5,"max":34221,"stddev":11402.3,"var":130012760.0,"ent":3.6,"data": [25199,25284,303,30105,3339,1074,34221,792,742,1850,1850,782,8352,423,28143,18603,6453,607,7069,119,26007,3,43,25894,1,59,186,154,696,4,1,0]},"pktlen": {"min":66,"avg":546.9,"max":1514,"stddev":584.9,"var":342142.3,"ent":4.2,"data": [78,74,66,583,66,1514,1514,66,1266,66,855,66,192,159,902,308,66,66,143,66,104,1119,1119,1514,66,66,66,724,66,1514,1514,1514]},"bins": {"c_to_s": [11,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"s_to_c": [3,0,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,1,0,0,0,0,0,0,0,2,0,0,0,0,1,0,0,0,0,0,0,0,6,0,0]},"directions": [0,1,0,0,1,1,1,0,1,0,1,0,0,0,0,1,0,1,1,0,0,1,1,1,0,0,0,1,0,1,1,1]},"ndpi": {"confidence": {"6":"DPI"},"proto":"TLS","proto_id":"91","encrypted":1,"breed":"Safe","category_id":5,"category":"Web"}}
+01688{"flow_event_id":5,"flow_event_name":"analyse","thread_id":0,"packet_id":32,"source":"tls_long_cert.pcap","alias":"nDPId-test","flow_id":1,"flow_state":"finished","flow_src_packets_processed":16,"flow_dst_packets_processed":16,"flow_first_seen":1553619078033240,"flow_src_last_pkt_time":1553619078157096,"flow_dst_last_pkt_time":1553619078157742,"flow_idle_time":3285032704,"flow_src_min_l4_payload_len":0,"flow_dst_min_l4_payload_len":0,"flow_src_max_l4_payload_len":836,"flow_dst_max_l4_payload_len":1448,"flow_src_tot_l4_payload_len":1610,"flow_dst_tot_l4_payload_len":13760,"midstream":0,"thread_ts_usec":1553619078157742,"l3_proto":"ip4","src_ip":"192.168.2.126","dst_ip":"104.111.215.93","src_port":60174,"dst_port":443,"l4_proto":"tcp","flow_datalink":1,"flow_max_packets":3,"data_analysis": {"iat": {"min":1,"avg":8011.5,"max":34221,"stddev":11402.3,"var":130012760.0,"ent":3.6,"data": [25199,25284,303,30105,3339,1074,34221,792,742,1850,1850,782,8352,423,28143,18603,6453,607,7069,119,26007,3,43,25894,1,59,186,154,696,4,1]},"pktlen": {"min":66,"avg":546.9,"max":1514,"stddev":584.9,"var":342142.3,"ent":4.2,"data": [78,74,66,583,66,1514,1514,66,1266,66,855,66,192,159,902,308,66,66,143,66,104,1119,1119,1514,66,66,66,724,66,1514,1514,1514]},"bins": {"c_to_s": [11,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"s_to_c": [3,0,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,1,0,0,0,0,0,0,0,2,0,0,0,0,1,0,0,0,0,0,0,0,6,0,0]},"directions": [0,1,0,0,1,1,1,0,1,0,1,0,0,0,0,1,0,1,1,0,0,1,1,1,0,0,0,1,0,1,1,1]},"ndpi": {"confidence": {"6":"DPI"},"proto":"TLS","proto_id":"91","encrypted":1,"breed":"Safe","category_id":5,"category":"Web"}}
 00909{"flow_event_id":2,"flow_event_name":"end","thread_id":0,"packet_id":182,"source":"tls_long_cert.pcap","alias":"nDPId-test","flow_id":1,"flow_state":"finished","flow_src_packets_processed":86,"flow_dst_packets_processed":96,"flow_first_seen":1553619078033240,"flow_src_last_pkt_time":1553619149347313,"flow_dst_last_pkt_time":1553619149372363,"flow_idle_time":3285032704,"flow_src_min_l4_payload_len":0,"flow_dst_min_l4_payload_len":0,"flow_src_max_l4_payload_len":836,"flow_dst_max_l4_payload_len":1448,"flow_src_tot_l4_payload_len":2858,"flow_dst_tot_l4_payload_len":102711,"midstream":0,"thread_ts_usec":1553619149372363,"l3_proto":"ip4","src_ip":"192.168.2.126","dst_ip":"104.111.215.93","src_port":60174,"dst_port":443,"l4_proto":"tcp","flow_datalink":1,"flow_max_packets":3,"ndpi": {"confidence": {"6":"DPI"},"proto":"TLS","proto_id":"91","encrypted":1,"breed":"Safe","category_id":5,"category":"Web"}}
 00570{"daemon_event_id":3,"daemon_event_name":"shutdown","thread_id":0,"packet_id":182,"source":"tls_long_cert.pcap","alias":"nDPId-test","packets-captured":182,"packets-processed":182,"total-skipped-flows":0,"total-l4-payload-len":105569,"total-not-detected-flows":0,"total-guessed-flows":0,"total-detected-flows":1,"total-detection-updates":2,"total-updates":0,"current-active-flows":0,"total-active-flows":1,"total-idle-flows":1,"total-compressions":0,"total-compression-diff":0,"current-compression-diff":0,"total-events-serialized":12,"global_ts_usec":1553619149372363}
 ~~~~~~~~~~~~~~~~~~~~ SUMMARY ~~~~~~~~~~~~~~~~~~~~
@@ -18,8 +18,8 @@
 ~~ total active/idle flows...: 1/1
 ~~ total timeout flows.......: 0
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-~~ total memory allocated....: 6083809 bytes
-~~ total memory freed........: 6083809 bytes
+~~ total memory allocated....: 6083805 bytes
+~~ total memory freed........: 6083805 bytes
 ~~ total allocations/frees...: 121732/121732
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 ~~ json string min len.......: 498 chars