Added nDPIsrvd-analysed to generate CSV files from analyse events.

 * nDPIsrvd.h: iterate over JSON arrays
 * nDPId: calculate l3 payload packet entropies for analysis

Signed-off-by: Toni Uhlig <matzeton@googlemail.com>

1 files changed, 4 insertions, 4 deletions

diff --git a/test/results/quickplay.pcap.out b/test/results/quickplay.pcap.out
index 2c4b3628a..bfe3d122a 100644
--- a/test/results/quickplay.pcap.out
+++ b/test/results/quickplay.pcap.out
@@ -63,7 +63,7 @@
 01111{"packet_event_id":2,"packet_event_name":"packet-flow","thread_id":0,"packet_id":36,"source":"quickplay.pcap","alias":"nDPId-test","flow_id":11,"flow_packet_id":3,"flow_src_last_pkt_time":1429000054688452,"flow_dst_last_pkt_time":1429000054555518,"flow_idle_time":3285032704,"pkt_oversize":false,"pkt_caplen":500,"pkt_type":2048,"pkt_l3_offset":16,"pkt_l4_offset":36,"pkt_len":500,"pkt_l4_len":464,"thread_ts_usec":1429000054688452,"pkt":"AAQCEgAAAAAAAAAAAAAIAEUAAeRjvEAAPwaG4wo2qfp4HCMoyykAUHABs6PLUc5cUBgk\/ZRuAABHRVQgL3NlZy92b2wxL3MvV2FybmVyL3FwbWV6emhhd2tkaWdpdGFsY29udGFnaW9uMjA1NDAzM2ZlYXR1cmVlbmdsaXNoMjBsdHJ0MjM5NzZmcHM3ODM0MTkyLzIwMTUtMDItMDIvU1RWODBSMTkyL3FwbWV6ei1IYXdrX0RpZ2l0YWxfQ09OVEFHSU9OXzIwNTQwMzNfRkVBVFVSRV9FTkdMSVNIXzJfMF9MVFJUXzIzOTc2ZnBzXzc4MzQxOTIubTJ0X1NUVjgwUjE5Mi0wMDIxLnRzIEhUVFAvMS4xDQpIb3N0OiB2b2Qtc2luZ3RlbGhhd2sucXVpY2twbGF5LmNvbQ0KQ29ubmVjdGlvbjoga2VlcC1hbGl2ZQ0KVXNlci1BZ2VudDogTW96aWxsYS81LjAgKExpbnV4OyBBbmRyb2lkIDQuNC40OyBNSSAzVyBCdWlsZC9LVFU4NFApIEFwcGxlV2ViS2l0LzUzNy4zNiAoS0hUTUwsIGxpa2UgR2Vja28pIFZlcnNpb24vNC4wIENocm9tZS8zMy4wLjAuMCBNb2JpbGUgU2FmYXJpLzUzNy4zNg0KDQo="}
 00792{"packet_event_id":2,"packet_event_name":"packet-flow","thread_id":0,"packet_id":37,"source":"quickplay.pcap","alias":"nDPId-test","flow_id":14,"flow_packet_id":2,"flow_src_last_pkt_time":1429000054595190,"flow_dst_last_pkt_time":1429000054967566,"flow_idle_time":3285032704,"pkt_oversize":false,"pkt_caplen":261,"pkt_type":2048,"pkt_l3_offset":16,"pkt_l4_offset":36,"pkt_len":261,"pkt_l4_len":225,"thread_ts_usec":1429000054967566,"pkt":"AAACEgAAAAAAAAAAAAAIAEUcAPUEEkAArQbHccvNgWUKNqn6AFCnCorJCJ8MOwSFUBgII8UCAABIVFRQLzEuMSAyMDAgT0sNCkNvbm5lY3Rpb246IGNsb3NlDQpDb250ZW50LVR5cGU6IGFwcGxpY2F0aW9uL29jdGV0LXN0cmVhbQ0KQ29udGVudC1EaXNwb3NpdGlvbjogYXR0YWNobWVudDsgZmlsZW5hbWU9bWljcm9tc2dyZXNwLmRhdA0KQ29udGVudC1MZW5ndGg6IDQ3DQoNCn5fAAAAAFUr0H3fAhACF0hkbD5sDN+EgwDvBAYGAAAXudj2eCNNjv4Uv\/n42\/lx"}
 00791{"packet_event_id":2,"packet_event_name":"packet-flow","thread_id":0,"packet_id":38,"source":"quickplay.pcap","alias":"nDPId-test","flow_id":13,"flow_packet_id":3,"flow_src_last_pkt_time":1429000052350287,"flow_dst_last_pkt_time":1429000055158240,"flow_idle_time":3285032704,"pkt_oversize":false,"pkt_caplen":261,"pkt_type":2048,"pkt_l3_offset":16,"pkt_l4_offset":36,"pkt_len":261,"pkt_l4_len":225,"thread_ts_usec":1429000055158240,"pkt":"AAACEgAAAAAAAAAAAAAIAEUcAPUJYkAArAas5svNl6AKNqn6AFDWZdcfCppPknoiUBkIIrzXAABIVFRQLzEuMSAyMDAgT0sNCkNvbm5lY3Rpb246IGNsb3NlDQpDb250ZW50LVR5cGU6IGFwcGxpY2F0aW9uL29jdGV0LXN0cmVhbQ0KQ29udGVudC1EaXNwb3NpdGlvbjogYXR0YWNobWVudDsgZmlsZW5hbWU9bWljcm9tc2dyZXNwLmRhdA0KQ29udGVudC1MZW5ndGg6IDQ3DQoNCn5fAAAAAFUr0H3fAhACF0hkbD5sDN+EgwD\/BAgIAACTADJ0e1hwz8xBqPPud44t"}
-01831{"flow_event_id":5,"flow_event_name":"analyse","thread_id":0,"packet_id":67,"source":"quickplay.pcap","alias":"nDPId-test","flow_id":11,"flow_state":"finished","flow_src_packets_processed":18,"flow_dst_packets_processed":14,"flow_first_seen":1429000052217627,"flow_src_last_pkt_time":1429000090450568,"flow_dst_last_pkt_time":1429000090229285,"flow_idle_time":3285032704,"flow_src_min_l4_payload_len":444,"flow_dst_min_l4_payload_len":0,"flow_src_max_l4_payload_len":531,"flow_dst_max_l4_payload_len":1400,"flow_src_tot_l4_payload_len":8360,"flow_dst_tot_l4_payload_len":10852,"midstream":1,"thread_ts_usec":1429000090450568,"l3_proto":"ip4","src_ip":"10.54.169.250","dst_ip":"120.28.35.40","src_port":52009,"dst_port":80,"l4_proto":"tcp","flow_datalink":113,"flow_max_packets":3,"data_analysis": {"iat": {"min":182557,"avg":2459503.2,"max":5871155,"stddev":1331263.2,"var":1772261736448.0,"ent":4.7,"data": [2337891,2470825,5776550,5871155,324615,2084534,1689148,182557,2170257,2013275,645600,519622,2223724,2353455,480927,4401947,3911834,3909668,3936554,2356476,2338349,2619995,2626526,2264068,2270477,2391541,2349518,2604523,2641967,2224884,2252137]},"pktlen": {"min":76,"avg":656.4,"max":1456,"stddev":347.9,"var":121006.6,"ent":4.8,"data": [500,1456,500,240,585,502,1248,585,502,854,587,76,504,1268,585,502,158,502,658,502,1124,502,1208,502,348,502,1456,502,962,502,580,502]},"bins": {"c_to_s": [0,0,0,0,0,0,0,0,0,0,0,0,0,13,1,0,4,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"s_to_c": [1,0,0,1,0,1,0,0,0,1,0,0,0,0,0,0,1,0,1,0,0,0,0,0,1,0,0,0,1,0,0,0,0,1,0,0,1,2,0,0,0,0,0,2,0,0,0,0]},"directions": [0,1,0,1,0,0,1,0,0,1,0,1,0,1,0,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0]},"ndpi": {"confidence": {"6":"DPI"},"proto":"HTTP","proto_id":"7","encrypted":0,"breed":"Acceptable","category_id":17,"category":"Streaming"}}
+02230{"flow_event_id":5,"flow_event_name":"analyse","thread_id":0,"packet_id":67,"source":"quickplay.pcap","alias":"nDPId-test","flow_id":11,"flow_state":"finished","flow_src_packets_processed":18,"flow_dst_packets_processed":14,"flow_first_seen":1429000052217627,"flow_src_last_pkt_time":1429000090450568,"flow_dst_last_pkt_time":1429000090229285,"flow_idle_time":3285032704,"flow_src_min_l4_payload_len":444,"flow_dst_min_l4_payload_len":0,"flow_src_max_l4_payload_len":531,"flow_dst_max_l4_payload_len":1400,"flow_src_tot_l4_payload_len":8360,"flow_dst_tot_l4_payload_len":10852,"midstream":1,"thread_ts_usec":1429000090450568,"l3_proto":"ip4","src_ip":"10.54.169.250","dst_ip":"120.28.35.40","src_port":52009,"dst_port":80,"l4_proto":"tcp","flow_datalink":113,"flow_max_packets":3,"data_analysis": {"iat": {"min":182557,"avg":2459503.2,"max":5871155,"stddev":1331263.2,"var":1772261736448.0,"ent":4.7,"data": [2337891,2470825,5776550,5871155,324615,2084534,1689148,182557,2170257,2013275,645600,519622,2223724,2353455,480927,4401947,3911834,3909668,3936554,2356476,2338349,2619995,2626526,2264068,2270477,2391541,2349518,2604523,2641967,2224884,2252137]},"pktlen": {"min":60,"avg":640.4,"max":1440,"stddev":347.9,"var":121006.6,"ent":4.8,"data": [484,1440,484,224,569,486,1232,569,486,838,571,60,488,1252,569,486,142,486,642,486,1108,486,1192,486,332,486,1440,486,946,486,564,486]},"bins": {"c_to_s": [0,0,0,0,0,0,0,0,0,0,0,0,0,13,1,0,4,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"s_to_c": [1,0,0,1,0,1,0,0,0,1,0,0,0,0,0,0,1,0,1,0,0,0,0,0,1,0,0,0,1,0,0,0,0,1,0,0,1,2,0,0,0,0,0,2,0,0,0,0]},"directions": [0,1,0,1,0,0,1,0,0,1,0,1,0,1,0,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0],"entropies": [5.927153111,7.868373871,5.963050365,7.051006317,5.916583061,5.928764343,7.836061001,5.947601795,5.927013874,7.735056400,5.960254192,4.985874176,5.956949711,7.848547459,5.950881958,5.944071770,6.557918549,5.946902752,7.695936680,5.966873169,7.840433598,5.939571857,7.838245869,5.963761330,7.329223633,5.943363190,7.857814789,5.947385788,7.759774208,5.933074474,7.621943474,5.938513279]},"ndpi": {"confidence": {"6":"DPI"},"proto":"HTTP","proto_id":"7","encrypted":0,"breed":"Acceptable","category_id":17,"category":"Streaming"}}
 00766{"flow_event_id":1,"flow_event_name":"new","thread_id":0,"packet_id":83,"source":"quickplay.pcap","alias":"nDPId-test","flow_id":15,"flow_state":"info","flow_src_packets_processed":1,"flow_dst_packets_processed":0,"flow_first_seen":1429000110390234,"flow_src_last_pkt_time":1429000110390234,"flow_dst_last_pkt_time":1429000110390234,"flow_idle_time":3285032704,"flow_src_min_l4_payload_len":625,"flow_dst_min_l4_payload_len":0,"flow_src_max_l4_payload_len":625,"flow_dst_max_l4_payload_len":0,"flow_src_tot_l4_payload_len":625,"flow_dst_tot_l4_payload_len":0,"midstream":1,"thread_ts_usec":1429000110390234,"l3_proto":"ip4","src_ip":"10.54.169.250","dst_ip":"203.205.147.215","src_port":35670,"dst_port":80,"l4_proto":"tcp","flow_datalink":113,"flow_max_packets":3}
 01356{"packet_event_id":2,"packet_event_name":"packet-flow","thread_id":0,"packet_id":83,"source":"quickplay.pcap","alias":"nDPId-test","flow_id":15,"flow_packet_id":1,"flow_src_last_pkt_time":1429000110390234,"flow_dst_last_pkt_time":1429000110390234,"flow_idle_time":3285032704,"pkt_oversize":false,"pkt_caplen":681,"pkt_type":2048,"pkt_l3_offset":16,"pkt_l4_offset":36,"pkt_len":681,"pkt_l4_len":645,"thread_ts_usec":1429000110390234,"pkt":"AAQCEgAAAAAAAAAAAAAIAEUAApm620AAPwZqrgo2qfrLzZPXi1YAUBkYU+pems3wUBgAbqLPAABQT1NUIGh0dHA6Ly9oa21pbm9yc2hvcnQud2VpeGluLnFxLmNvbS9jZ2ktYmluL21pY3JvbXNnLWJpbi9ydGt2cmVwb3J0IEhUVFAvMS4xDQpBY2NlcHQ6ICovKg0KQ2FjaGUtQ29udHJvbDogbm8tY2FjaGUNCkNvbm5lY3Rpb246IGNsb3NlDQpDb250ZW50LUxlbmd0aDogMzU1DQpDb250ZW50LVR5cGU6IGFwcGxpY2F0aW9uL29jdGV0LXN0cmVhbQ0KSG9zdDogaGttaW5vcnNob3J0LndlaXhpbi5xcS5jb20NClVzZXItQWdlbnQ6IE1pY3JvTWVzc2VuZ2VyIENsaWVudA0KDQqNXyYBAEFVK9B93wIQAhdIZGw+bAzfhIMAzAXaBLgCxN0BAsPj4k0n0ICdjYK52ViOWnOC4Vzgxi7+iegOsMW+oW6QdEAHg+UlyxaSBb9\/s0oeR4gum6gk+uWhqjv3Tkoz3jpOxZ3uqg5IoeAevVK78mE+75Mm5QEXaL\/24wa8I4nsiJTVEr54yg9WsIjA1I\/cd65YM57jS4+t1kJ\/xpqwwPsMfqK2G34N85Xo0uWP1F2PyLEjHiJZyK4xRu\/XYVzahdDn1vQRPtqQ3i2o6ggKNGN3kBkFa6C2GO0zTqwt7XUYqb0ppGq3KKIyPCtrTg5YICuEsfTDMTLer3J067M5VD93Ij+RkxqqGFN9+gvu+C\/smM0OksnEYsvtVnkr65ZF5Pk4qVPYHRDIlRcRHe0XzckIkJitYHFr8VSN2R6GxFfZK0YtMPQdmLxH6qLecheL3Cuuz7XcYpBc6JGpDIih+q4v"}
 01248{"flow_event_id":7,"flow_event_name":"detected","thread_id":0,"packet_id":83,"source":"quickplay.pcap","alias":"nDPId-test","flow_id":15,"flow_state":"info","flow_src_packets_processed":1,"flow_dst_packets_processed":0,"flow_first_seen":1429000110390234,"flow_src_last_pkt_time":1429000110390234,"flow_dst_last_pkt_time":1429000110390234,"flow_idle_time":3285032704,"flow_src_min_l4_payload_len":625,"flow_dst_min_l4_payload_len":0,"flow_src_max_l4_payload_len":625,"flow_dst_max_l4_payload_len":0,"flow_src_tot_l4_payload_len":625,"flow_dst_tot_l4_payload_len":0,"midstream":1,"thread_ts_usec":1429000110390234,"l3_proto":"ip4","src_ip":"10.54.169.250","dst_ip":"203.205.147.215","src_port":35670,"dst_port":80,"l4_proto":"tcp","ndpi": {"flow_risk": {"5": {"risk":"Known Proto on Non Std Port","severity":"Medium","risk_score": {"total":260,"client":230,"server":30}}},"confidence": {"6":"DPI"},"proto":"HTTP_Proxy.QQ","proto_id":"131.48","encrypted":0,"breed":"Fun","category_id":9,"category":"Chat","hostname":"hkminorshort.weixin.qq.com","http": {"url":"http:\/\/hkminorshort.weixin.qq.com\/cgi-bin\/micromsg-bin\/rtkvreport","code":0,"content_type":"","user_agent":"MicroMessenger Client","request_content_type":"application\/octet-stream"}}}
@@ -127,9 +127,9 @@
 ~~ total active/idle flows...: 21/21
 ~~ total timeout flows.......: 0
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-~~ total memory allocated....: 6078932 bytes
-~~ total memory freed........: 6078932 bytes
-~~ total allocations/frees...: 121946/121946
+~~ total memory allocated....: 6081788 bytes
+~~ total memory freed........: 6081788 bytes
+~~ total allocations/frees...: 121967/121967
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 ~~ json string min len.......: 494 chars
 ~~ json string max len.......: 2426 chars