Improved the point of time when to append the raw packet base64 data to the serializer.

 * nDPId-test: Increased the max-packets-per-flow-to-send from 3 to 5.
   This is quite useful for TCP as the first 3 packets are usually part of the three-way-handshake.

Signed-off-by: Toni Uhlig <matzeton@googlemail.com>

1 files changed, 10 insertions, 9 deletions

diff --git a/test/results/http_on_sip_port.pcap.out b/test/results/http_on_sip_port.pcap.out
index 1c3445cc4..a9bd4a8e9 100644
--- a/test/results/http_on_sip_port.pcap.out
+++ b/test/results/http_on_sip_port.pcap.out
@@ -1,12 +1,13 @@
-00496{"daemon_event_id":1,"daemon_event_name":"init","thread_id":0,"packet_id":0,"source":"http_on_sip_port.pcap","alias":"nDPId-test","max-flows-per-thread":2048,"max-idle-flows-per-thread":64,"reader-thread-count":1,"flow-scan-interval":10000000,"generic-max-idle-time":600000000,"icmp-max-idle-time":120000000,"udp-max-idle-time":180000000,"tcp-max-idle-time":3265032704,"max-packets-per-flow-to-send":3,"max-packets-per-flow-to-process":32,"max-packets-per-flow-to-analyse":32,"global_ts_usec":0}
+00496{"daemon_event_id":1,"daemon_event_name":"init","thread_id":0,"packet_id":0,"source":"http_on_sip_port.pcap","alias":"nDPId-test","max-flows-per-thread":2048,"max-idle-flows-per-thread":64,"reader-thread-count":1,"flow-scan-interval":10000000,"generic-max-idle-time":600000000,"icmp-max-idle-time":120000000,"udp-max-idle-time":180000000,"tcp-max-idle-time":3265032704,"max-packets-per-flow-to-send":5,"max-packets-per-flow-to-process":32,"max-packets-per-flow-to-analyse":32,"global_ts_usec":0}
 00559{"daemon_event_id":4,"daemon_event_name":"status","thread_id":0,"packet_id":1,"source":"http_on_sip_port.pcap","alias":"nDPId-test","packets-captured":1,"packets-processed":0,"total-skipped-flows":0,"total-l4-payload-len":0,"total-not-detected-flows":0,"total-guessed-flows":0,"total-detected-flows":0,"total-detection-updates":0,"total-updates":0,"current-active-flows":0,"total-active-flows":0,"total-idle-flows":0,"total-compressions":0,"total-compression-diff":0,"current-compression-diff":0,"total-events-serialized":2,"global_ts_usec":1618744016209720}
-00761{"flow_event_id":1,"flow_event_name":"new","thread_id":0,"packet_id":1,"source":"http_on_sip_port.pcap","alias":"nDPId-test","flow_id":1,"flow_state":"info","flow_src_packets_processed":1,"flow_dst_packets_processed":0,"flow_first_seen":1618744016209720,"flow_src_last_pkt_time":1618744016209720,"flow_dst_last_pkt_time":1618744016209720,"flow_idle_time":3285032704,"flow_src_min_l4_payload_len":0,"flow_dst_min_l4_payload_len":0,"flow_src_max_l4_payload_len":0,"flow_dst_max_l4_payload_len":0,"flow_src_tot_l4_payload_len":0,"flow_dst_tot_l4_payload_len":0,"midstream":0,"thread_ts_usec":1618744016209720,"l3_proto":"ip4","src_ip":"82.178.111.221","dst_ip":"45.58.148.2","src_port":5060,"dst_port":8888,"l4_proto":"tcp","flow_datalink":1,"flow_max_packets":3}
-00547{"packet_event_id":2,"packet_event_name":"packet-flow","thread_id":0,"packet_id":1,"source":"http_on_sip_port.pcap","alias":"nDPId-test","flow_id":1,"flow_packet_id":1,"flow_src_last_pkt_time":1618744016209720,"flow_dst_last_pkt_time":1618744016209720,"flow_idle_time":3285032704,"pkt_oversize":false,"pkt_caplen":74,"pkt_type":2048,"pkt_l3_offset":14,"pkt_l4_offset":34,"pkt_len":74,"pkt_l4_len":40,"thread_ts_usec":1618744016209720,"pkt":"AAAAAAAAAAsAxhT1CABFAAA8sxJAAD4GBd5Ssm\/dLTqUAhPEIrha1ycbAAAAAKAC\/\/9M3wAAAgQFUAQCCAoQxK6EAAAAAAEDAwk="}
-00536{"packet_event_id":2,"packet_event_name":"packet-flow","thread_id":0,"packet_id":2,"source":"http_on_sip_port.pcap","alias":"nDPId-test","flow_id":1,"flow_packet_id":2,"flow_src_last_pkt_time":1618744016209720,"flow_dst_last_pkt_time":1618744016342703,"flow_idle_time":3285032704,"pkt_oversize":false,"pkt_caplen":66,"pkt_type":2048,"pkt_l3_offset":14,"pkt_l4_offset":34,"pkt_len":66,"pkt_l4_len":32,"thread_ts_usec":1618744016342703,"pkt":"AAAAAAAAAAUAQPTMCABFAAA0AABAADMGw\/gtOpQCUrJv3SK4E8QPDztmWtcnHIAS\/\/\/oTwAAAgQFtAEBBAIBAwMI"}
-00820{"packet_event_id":2,"packet_event_name":"packet-flow","thread_id":0,"packet_id":3,"source":"http_on_sip_port.pcap","alias":"nDPId-test","flow_id":1,"flow_packet_id":3,"flow_src_last_pkt_time":1618744016398438,"flow_dst_last_pkt_time":1618744016342703,"flow_idle_time":3285032704,"pkt_oversize":false,"pkt_caplen":277,"pkt_type":2048,"pkt_l3_offset":14,"pkt_l4_offset":34,"pkt_len":277,"pkt_l4_len":243,"thread_ts_usec":1618744016398438,"pkt":"AAAAAAAAAAsAxhT1CABFAAEHsxRAAD4GBRFSsm\/dLTqUAhPEIrha1yccDw87Z1AYAKwvFgAAR0VUIC9zdGFyLTEyMzQ1Ni9pbmRleC5tM3U4P3Rva2VuPTg5YjE5OGI4ODQ0ODI0Y2ExNWI4YjM3OWMyNmZjMWI3ZGZjYmEzNjgtNUtVSlRKNVk3M0FHSUFPVi0xNjE4NzUzMTc0LTE2MTg3NDIzNzQgSFRUUC8xLjENClVzZXItQWdlbnQ6IGV4b3BsYXllci1jb2RlbGFiDQpIb3N0OiA0NS41OC4xNDguMg0KQ29ubmVjdGlvbjogS2VlcC1BbGl2ZQ0KQWNjZXB0LUVuY29kaW5nOiBnemlwDQoNCg=="}
+00761{"flow_event_id":1,"flow_event_name":"new","thread_id":0,"packet_id":1,"source":"http_on_sip_port.pcap","alias":"nDPId-test","flow_id":1,"flow_state":"info","flow_src_packets_processed":1,"flow_dst_packets_processed":0,"flow_first_seen":1618744016209720,"flow_src_last_pkt_time":1618744016209720,"flow_dst_last_pkt_time":1618744016209720,"flow_idle_time":3285032704,"flow_src_min_l4_payload_len":0,"flow_dst_min_l4_payload_len":0,"flow_src_max_l4_payload_len":0,"flow_dst_max_l4_payload_len":0,"flow_src_tot_l4_payload_len":0,"flow_dst_tot_l4_payload_len":0,"midstream":0,"thread_ts_usec":1618744016209720,"l3_proto":"ip4","src_ip":"82.178.111.221","dst_ip":"45.58.148.2","src_port":5060,"dst_port":8888,"l4_proto":"tcp","flow_datalink":1,"flow_max_packets":5}
+00526{"packet_event_id":2,"packet_event_name":"packet-flow","thread_id":0,"packet_id":1,"source":"http_on_sip_port.pcap","alias":"nDPId-test","flow_id":1,"flow_packet_id":1,"flow_src_last_pkt_time":1618744016209720,"flow_dst_last_pkt_time":1618744016209720,"flow_idle_time":3285032704,"pkt_caplen":74,"pkt_type":2048,"pkt_l3_offset":14,"pkt_l4_offset":34,"pkt_len":74,"pkt_l4_len":40,"thread_ts_usec":1618744016209720,"pkt":"AAAAAAAAAAsAxhT1CABFAAA8sxJAAD4GBd5Ssm\/dLTqUAhPEIrha1ycbAAAAAKAC\/\/9M3wAAAgQFUAQCCAoQxK6EAAAAAAEDAwk="}
+00515{"packet_event_id":2,"packet_event_name":"packet-flow","thread_id":0,"packet_id":2,"source":"http_on_sip_port.pcap","alias":"nDPId-test","flow_id":1,"flow_packet_id":2,"flow_src_last_pkt_time":1618744016209720,"flow_dst_last_pkt_time":1618744016342703,"flow_idle_time":3285032704,"pkt_caplen":66,"pkt_type":2048,"pkt_l3_offset":14,"pkt_l4_offset":34,"pkt_len":66,"pkt_l4_len":32,"thread_ts_usec":1618744016342703,"pkt":"AAAAAAAAAAUAQPTMCABFAAA0AABAADMGw\/gtOpQCUrJv3SK4E8QPDztmWtcnHIAS\/\/\/oTwAAAgQFtAEBBAIBAwMI"}
+00799{"packet_event_id":2,"packet_event_name":"packet-flow","thread_id":0,"packet_id":3,"source":"http_on_sip_port.pcap","alias":"nDPId-test","flow_id":1,"flow_packet_id":3,"flow_src_last_pkt_time":1618744016398438,"flow_dst_last_pkt_time":1618744016342703,"flow_idle_time":3285032704,"pkt_caplen":277,"pkt_type":2048,"pkt_l3_offset":14,"pkt_l4_offset":34,"pkt_len":277,"pkt_l4_len":243,"thread_ts_usec":1618744016398438,"pkt":"AAAAAAAAAAsAxhT1CABFAAEHsxRAAD4GBRFSsm\/dLTqUAhPEIrha1yccDw87Z1AYAKwvFgAAR0VUIC9zdGFyLTEyMzQ1Ni9pbmRleC5tM3U4P3Rva2VuPTg5YjE5OGI4ODQ0ODI0Y2ExNWI4YjM3OWMyNmZjMWI3ZGZjYmEzNjgtNUtVSlRKNVk3M0FHSUFPVi0xNjE4NzUzMTc0LTE2MTg3NDIzNzQgSFRUUC8xLjENClVzZXItQWdlbnQ6IGV4b3BsYXllci1jb2RlbGFiDQpIb3N0OiA0NS41OC4xNDguMg0KQ29ubmVjdGlvbjogS2VlcC1BbGl2ZQ0KQWNjZXB0LUVuY29kaW5nOiBnemlwDQoNCg=="}
 01377{"flow_event_id":7,"flow_event_name":"detected","thread_id":0,"packet_id":3,"source":"http_on_sip_port.pcap","alias":"nDPId-test","flow_id":1,"flow_state":"info","flow_src_packets_processed":2,"flow_dst_packets_processed":1,"flow_first_seen":1618744016209720,"flow_src_last_pkt_time":1618744016398438,"flow_dst_last_pkt_time":1618744016342703,"flow_idle_time":3285032704,"flow_src_min_l4_payload_len":0,"flow_dst_min_l4_payload_len":0,"flow_src_max_l4_payload_len":223,"flow_dst_max_l4_payload_len":0,"flow_src_tot_l4_payload_len":223,"flow_dst_tot_l4_payload_len":0,"midstream":0,"thread_ts_usec":1618744016398438,"l3_proto":"ip4","src_ip":"82.178.111.221","dst_ip":"45.58.148.2","src_port":5060,"dst_port":8888,"l4_proto":"tcp","ndpi": {"flow_risk": {"5": {"risk":"Known Proto on Non Std Port","severity":"Medium","risk_score": {"total":160,"client":140,"server":20}},"12": {"risk":"HTTP Numeric IP Address","severity":"Low","risk_score": {"total":300,"client":270,"server":30}}},"confidence": {"6":"DPI"},"proto":"HTTP","proto_id":"7","proto_by_ip":"Unknown","proto_by_ip_id":0,"encrypted":0,"breed":"Acceptable","category_id":5,"category":"Web","hostname":"45.58.148.2","http": {"url":"45.58.148.2\/star-123456\/index.m3u8?token=89b198b8844824ca15b8b379c26fc1b7dfcba368-5KUJTJ5Y73AGIAOV-1618753174-1618742374","code":0,"content_type":"","user_agent":"exoplayer-codelab"}}}
-01196{"flow_event_id":3,"flow_event_name":"idle","thread_id":0,"packet_id":4,"source":"http_on_sip_port.pcap","alias":"nDPId-test","flow_id":1,"flow_state":"finished","flow_src_packets_processed":2,"flow_dst_packets_processed":2,"flow_first_seen":1618744016209720,"flow_src_last_pkt_time":1618744016398438,"flow_dst_last_pkt_time":1618744016532140,"flow_idle_time":3285032704,"flow_src_min_l4_payload_len":0,"flow_dst_min_l4_payload_len":0,"flow_src_max_l4_payload_len":223,"flow_dst_max_l4_payload_len":1360,"flow_src_tot_l4_payload_len":223,"flow_dst_tot_l4_payload_len":1360,"midstream":0,"thread_ts_usec":1618744016532140,"l3_proto":"ip4","src_ip":"82.178.111.221","dst_ip":"45.58.148.2","src_port":5060,"dst_port":8888,"l4_proto":"tcp","flow_datalink":1,"flow_max_packets":3,"ndpi": {"flow_risk": {"5": {"risk":"Known Proto on Non Std Port","severity":"Medium","risk_score": {"total":160,"client":140,"server":20}},"12": {"risk":"HTTP Numeric IP Address","severity":"Low","risk_score": {"total":300,"client":270,"server":30}}},"confidence": {"6":"DPI"},"proto":"HTTP","proto_id":"7","proto_by_ip":"Unknown","proto_by_ip_id":0,"encrypted":0,"breed":"Acceptable","category_id":5,"category":"Web"}}
-00564{"daemon_event_id":3,"daemon_event_name":"shutdown","thread_id":0,"packet_id":4,"source":"http_on_sip_port.pcap","alias":"nDPId-test","packets-captured":4,"packets-processed":4,"total-skipped-flows":0,"total-l4-payload-len":1583,"total-not-detected-flows":0,"total-guessed-flows":0,"total-detected-flows":1,"total-detection-updates":0,"total-updates":0,"current-active-flows":0,"total-active-flows":1,"total-idle-flows":1,"total-compressions":0,"total-compression-diff":0,"current-compression-diff":0,"total-events-serialized":9,"global_ts_usec":1618744016532140}
+02317{"packet_event_id":2,"packet_event_name":"packet-flow","thread_id":0,"packet_id":4,"source":"http_on_sip_port.pcap","alias":"nDPId-test","flow_id":1,"flow_packet_id":4,"flow_src_last_pkt_time":1618744016398438,"flow_dst_last_pkt_time":1618744016532140,"flow_idle_time":3285032704,"pkt_caplen":1414,"pkt_type":2048,"pkt_l3_offset":14,"pkt_l4_offset":34,"pkt_len":1414,"pkt_l4_len":1380,"thread_ts_usec":1618744016532140,"pkt":"AAAAAAAAAAUAQPTMCABFAAV45dJAADMG2OEtOpQCUrJv3SK4E8QPDztnWtcn+1AQAQUxMQAASFRUUC8xLjEgNDAzIEZvcmJpZGRlbg0KQ29ubmVjdGlvbjoga2VlcC1hbGl2ZQ0KRGF0ZTogU3VuLCAxOCBBcHIgMjAyMSAxMTowNDo0MCBHTVQNCkNvbnRlbnQtTGVuZ3RoOiAxMTY3DQpTZXJ2ZXI6IEZsdXNzb25pYw0KWC1Sb3V0ZS1UaW1lOiAxMTINClgtUnVuLVRpbWU6IDI0MQ0KQWNjZXNzLUNvbnRyb2wtQWxsb3ctT3JpZ2luOiAqDQpBY2Nlc3MtQ29udHJvbC1BbGxvdy1NZXRob2RzOiBHRVQsIFBVVCwgREVMRVRFLCBPUFRJT05TDQpBY2Nlc3MtQ29udHJvbC1FeHBvc2UtSGVhZGVyczogU2VydmVyLCByYW5nZSwgWC1SdW4tVGltZSwgQ29udGVudC1MZW5ndGgsIExvY2F0aW9uDQpBY2Nlc3MtQ29udHJvbC1BbGxvdy1IZWFkZXJzOiB4LXZzYWFzLXNlc3Npb24sIHgtbm8tcmVkaXJlY3QsIG9yaWdpbiwgYXV0aG9yaXphdGlvbiwgeC1yZWFsLWlwLCBhY2NlcHQsIHJhbmdlDQpYLURlbnktUmVhc29uOiBjYWNoZWRfbmVnYXRpdmUNCg0KPCFET0NUWVBFIGh0bWw+CjxodG1sPgo8aGVhZD4KICA8dGl0bGU+Rmx1c3NvbmljIHN0cmVhbWluZyBzZXJ2ZXI8L3RpdGxlPgogIDxtZXRhIG5hbWU9ImZyYWdtZW50IiBjb250ZW50PSIhIiAvPgogIDxtZXRhIG5hbWU9InZpZXdwb3J0IiBjb250ZW50PSJ3aWR0aD1kZXZpY2Utd2lkdGgsIGluaXRpYWwtc2NhbGU9MS4wIj4KICA8bWV0YSBjaGFyc2V0PSJ1dGY4Ij4KPC9oZWFkPgo8Ym9keT4KCjxzdHlsZSB0eXBlPSJ0ZXh0L2NzcyI+CmJvZHkgewogIGZvbnQtZmFtaWx5OiAiSGVsdmV0aWNhIE5ldWUiLCBIZWx2ZXRpY2EsIEFyaWFsLCBzYW5zLXNlcmlmOwogIGZvbnQtc2l6ZTogMTRweDsKICBsaW5lLWhlaWdodDogMS40Mjg1NzE0Mjk7CiAgY29sb3I6ICMzMzMzMzM7Cn0KLmNvbnRhaW5lciB7CiAgbWFyZ2luLXJpZ2h0OiBhdXRvOwogIG1hcmdpbi1sZWZ0OiBhdXRvOwogIHBhZGRpbmctbGVmdDogMTVweDsKICBwYWRkaW5nLXJpZ2h0OiAxNXB4Owp9CkBtZWRpYSAobWluLXdpZHRoOiA3NjhweCkgeyAuY29udGFpbmVyIHsgbWF4LXdpZHRoOiA3NTBweDsgfSB9CkBtZWRpYSAobWluLXdpZHRoOiA5OTJweCkgeyAuY29udGFpbmVyIHsgbWF4LXdpZHRoOiA5NzBweDsgfSB9CkBtZWRpYSAobWluLXdpZHRoOiAxMjAwcHgpIHsgLmNvbnRhaW5lciB7IG1heC13aWR0aDogMTE3MHB4OyB9IH0KLnBhZ2UtaGVhZGVyIHsKICBwYWRkaW5nLWJvdHRvbTogOXB4OwogIG1hcmdpbjogNDBweCAwIDIwcHg7CiAgYm9yZGVyLWJvdHRvbTogMXB4IHNvbGlkICNlZWVlZWU7Cn0KaDEgewogIGZvbnQtc2l6ZTogMzZweDsKICBtYXJnaW4tdG9wOiAyMHB4OwogIG1hcmdpbi1ib3R0b206IDEwcHg7CiAgZm9udC13ZWlnaHQ6IDUwMDsKICBsaW5lLWhlaWdodA=="}
+01196{"flow_event_id":3,"flow_event_name":"idle","thread_id":0,"packet_id":4,"source":"http_on_sip_port.pcap","alias":"nDPId-test","flow_id":1,"flow_state":"finished","flow_src_packets_processed":2,"flow_dst_packets_processed":2,"flow_first_seen":1618744016209720,"flow_src_last_pkt_time":1618744016398438,"flow_dst_last_pkt_time":1618744016532140,"flow_idle_time":3285032704,"flow_src_min_l4_payload_len":0,"flow_dst_min_l4_payload_len":0,"flow_src_max_l4_payload_len":223,"flow_dst_max_l4_payload_len":1360,"flow_src_tot_l4_payload_len":223,"flow_dst_tot_l4_payload_len":1360,"midstream":0,"thread_ts_usec":1618744016532140,"l3_proto":"ip4","src_ip":"82.178.111.221","dst_ip":"45.58.148.2","src_port":5060,"dst_port":8888,"l4_proto":"tcp","flow_datalink":1,"flow_max_packets":5,"ndpi": {"flow_risk": {"5": {"risk":"Known Proto on Non Std Port","severity":"Medium","risk_score": {"total":160,"client":140,"server":20}},"12": {"risk":"HTTP Numeric IP Address","severity":"Low","risk_score": {"total":300,"client":270,"server":30}}},"confidence": {"6":"DPI"},"proto":"HTTP","proto_id":"7","proto_by_ip":"Unknown","proto_by_ip_id":0,"encrypted":0,"breed":"Acceptable","category_id":5,"category":"Web"}}
+00565{"daemon_event_id":3,"daemon_event_name":"shutdown","thread_id":0,"packet_id":4,"source":"http_on_sip_port.pcap","alias":"nDPId-test","packets-captured":4,"packets-processed":4,"total-skipped-flows":0,"total-l4-payload-len":1583,"total-not-detected-flows":0,"total-guessed-flows":0,"total-detected-flows":1,"total-detection-updates":0,"total-updates":0,"current-active-flows":0,"total-active-flows":1,"total-idle-flows":1,"total-compressions":0,"total-compression-diff":0,"current-compression-diff":0,"total-events-serialized":10,"global_ts_usec":1618744016532140}
 ~~~~~~~~~~~~~~~~~~~~ SUMMARY ~~~~~~~~~~~~~~~~~~~~
 ~~ packets captured/processed: 4/4
 ~~ skipped flows.............: 0
@@ -20,5 +21,5 @@
 ~~ total allocations/frees...: 122446/122446
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 ~~ json string min len.......: 501 chars
-~~ json string max len.......: 1382 chars
-~~ json string avg len.......: 902 chars
+~~ json string max len.......: 2322 chars
+~~ json string avg len.......: 1332 chars