Added nDPId semantic validation test.

 * fixed inconsistent processing of remaining flows during nDPId shutdown phase
 * fixed multiple `detected' flow events
   (instead only `detection-update' flow events can occur after a `detected' flow event)
 * fixed nDPIsrvd.py invalid message buffer handling
 * improved run_tests.sh so only valid pcap capture files are getting processed
   (and some more cosmetics + logging)

Signed-off-by: Toni Uhlig <matzeton@googlemail.com>

20 files changed, 381 insertions, 93 deletions

diff --git a/dependencies/nDPIsrvd.py b/dependencies/nDPIsrvd.py
index 643a1a7e5..7024d8f22 100644
--- a/dependencies/nDPIsrvd.py
+++ b/dependencies/nDPIsrvd.py
@@ -183,17 +183,19 @@ class nDPIsrvdSocket:
         if len(self.buffer) == NETWORK_BUFFER_MAX_SIZE:
             raise BufferCapacityReached(len(self.buffer), NETWORK_BUFFER_MAX_SIZE)
 
+        connection_finished = False
         try:
             recvd = self.sock.recv(NETWORK_BUFFER_MAX_SIZE - len(self.buffer))
         except ConnectionResetError:
-            raise SocketConnectionBroken()
-
+            connection_finished = True
+            recvd = bytes()
         if len(recvd) == 0:
-            raise SocketConnectionBroken()
+            connection_finished = True
+
         self.buffer += recvd
 
         new_data_avail = False
-        while self.msglen + self.digitlen < len(self.buffer):
+        while self.msglen + self.digitlen <= len(self.buffer):
 
             if self.msglen == 0:
                 starts_with_digits = re.match(r'(^\d+){', self.buffer[:NETWORK_BUFFER_MIN_SIZE].decode(errors='strict'))
@@ -213,6 +215,9 @@ class nDPIsrvdSocket:
                 self.msglen = 0
                 self.digitlen = 0
 
+        if connection_finished is True:
+            raise SocketConnectionBroken()
+
         return new_data_avail
 
     def parse(self, callback, global_user_data):
@@ -230,11 +235,20 @@ class nDPIsrvdSocket:
         return retval
 
     def loop(self, callback, global_user_data):
+        throw_ex = None
+
         while True:
-            if self.receive() > 0:
-                if self.parse(callback, global_user_data) is False:
-                    raise CallbackReturnedFalse()
-                    break;
+            bytes_recv = 0
+            try:
+                bytes_recv = self.receive()
+            except Exception as err:
+                throw_ex = err
+
+            if self.parse(callback, global_user_data) is False:
+                raise CallbackReturnedFalse()
+
+            if throw_ex is not None:
+                raise throw_ex
 
 class PcapPacket:
     def __init__(self):
diff --git a/examples/README.md b/examples/README.md
index 676e9a9d2..05552b008 100644
--- a/examples/README.md
+++ b/examples/README.md
@@ -19,7 +19,7 @@ Tiny nDPId json dumper. Does not provide any useful funcationality besides dumpi
 
 ## go-dashboard
 
-A discontinued tty/ncurses nDPId dashboard. I've figured out that Go + NCurses is a bad idea.
+A discontinued tty UI nDPId dashboard. I've figured out that Go + UI is a bad idea, in particular if performance is a concern.
 
 ## py-flow-info
 
@@ -41,3 +41,6 @@ Captures and saves risky flows to a PCAP file.
 
 Validate nDPId JSON strings against pre-defined JSON schema's.
 See `schema/`.
+
+## py-semantic-validation
+Validate nDPId JSON strings against internal event semantics.
diff --git a/examples/py-schema-validation/py-schema-validation.py b/examples/py-schema-validation/py-schema-validation.py
index defba96e9..6273a5aa5 100755
--- a/examples/py-schema-validation/py-schema-validation.py
+++ b/examples/py-schema-validation/py-schema-validation.py
@@ -48,3 +48,5 @@ if __name__ == '__main__':
         nsock.loop(onJsonLineRecvd, Stats())
     except nDPIsrvd.SocketConnectionBroken as err:
         sys.stderr.write('\n{}\n'.format(err))
+    except KeyboardInterrupt:
+        print()
diff --git a/examples/py-semantic-validation/py-semantic-validation.py b/examples/py-semantic-validation/py-semantic-validation.py
new file mode 100755
index 000000000..d4423467e
--- /dev/null
+++ b/examples/py-semantic-validation/py-semantic-validation.py
@@ -0,0 +1,182 @@
+#!/usr/bin/env python3
+
+import os
+import sys
+
+sys.path.append(os.path.dirname(sys.argv[0]) + '/../share/nDPId')
+sys.path.append(os.path.dirname(sys.argv[0]) + '/../usr/share/nDPId')
+try:
+    import nDPIsrvd
+    from nDPIsrvd import nDPIsrvdSocket, TermColor
+except ImportError:
+    sys.path.append(os.path.dirname(sys.argv[0]) + '/../../dependencies')
+    import nDPIsrvd
+    from nDPIsrvd import nDPIsrvdSocket, TermColor
+
+global lowest_flow_id_for_new_flow
+lowest_flow_id_for_new_flow = 0
+
+class Stats:
+    event_counter   = dict()
+
+    lines_processed = 0
+    print_dot_every = 10
+    print_nmb_every = print_dot_every * 5
+
+    def resetEventCounter(self):
+        keys = ['init','reconnect','shutdown', \
+                'new','end','idle','guessed','detected','detection-update','not-detected', \
+                'packet', 'packet-flow']
+        for k in keys:
+            self.event_counter[k] = 0
+
+    def incrementEventCounter(self, json_dict):
+        try:
+            if 'daemon_event_name' in json_dict:
+                self.event_counter[json_dict['daemon_event_name']] += 1
+            if 'flow_event_name' in json_dict:
+                self.event_counter[json_dict['flow_event_name']] += 1
+            if 'packet_event_name' in json_dict:
+                self.event_counter[json_dict['packet_event_name']] += 1
+        except KeyError as e:
+            raise RuntimeError('Semantic validation failed for event counter '
+                               'which received an invalid key: {}'.format(str(e)))
+
+    def verifyEventCounter(self):
+        if self.event_counter['shutdown'] != self.event_counter['init'] or self.event_counter['init'] == 0:
+            return False
+        if self.event_counter['new'] != self.event_counter['end'] + self.event_counter['idle']:
+            return False
+        if self.event_counter['new'] < self.event_counter['detected'] + self.event_counter['not-detected']:
+            return False
+        if self.event_counter['new'] < self.event_counter['guessed'] + self.event_counter['not-detected']:
+            return False
+
+        return True
+
+    def getEventCounterStr(self):
+        keys = [ [ 'init','reconnect','shutdown' ], \
+                 [ 'new','end','idle' ], \
+                 [ 'guessed','detected','detection-update','not-detected' ], \
+                 [ 'packet', 'packet-flow' ] ]
+        retval = str()
+        retval += '-' * 98 + '--\n'
+        for klist in keys:
+            for k in klist:
+                retval += '| {:<16}: {:<4} '.format(k, self.event_counter[k])
+            retval += '\n--' + '-' * 98 + '\n'
+        return retval
+
+    def __init__(self):
+        self.resetEventCounter()
+
+class SemanticValidationException(Exception):
+    def __init__(self, current_flow, text):
+        self.text = text
+        self.current_flow = current_flow
+    def __str__(self):
+        if self.current_flow is None:
+            return '{}'.format(self.text)
+        else:
+            return 'Flow ID {}: {}'.format(self.current_flow.flow_id, self.text)
+
+def onJsonLineRecvd(json_dict, current_flow, global_user_data):
+    global lowest_flow_id_for_new_flow
+    stats = global_user_data
+    stats.incrementEventCounter(json_dict)
+
+    try:
+        semdict = current_flow.semdict
+    except AttributeError:
+        try:
+            semdict = current_flow.semdict = dict()
+        except AttributeError:
+            semdict = dict()
+
+    if 'current_flow' in semdict:
+        if semdict['current_flow'] != current_flow:
+            raise SemanticValidationException(current_flow,
+                                              'Semantic dictionary flow reference != current flow reference: ' \
+                                              '{} != {}'.format(semdict['current_flow'], current_flow))
+    else:
+        semdict['current_flow'] = current_flow
+
+    if current_flow is not None:
+        if 'flow_id' in semdict:
+            if semdict['flow_id'] != current_flow.flow_id or \
+               semdict['flow_id'] != json_dict['flow_id']:
+                raise SemanticValidationException(current_flow,
+                                                  'Semantic dictionary flow id != current flow id != JSON dictionary flow id: ' \
+                                                  '{} != {} != {}'.format(semdict['flow_id'], \
+                                                  current_flow.flow_id, json_dict['flow_id']))
+        else:
+            if json_dict['flow_id'] != current_flow.flow_id:
+                raise SemanticValidationException(current_flow,
+                                                  'JSON dictionary flow id != current flow id: ' \
+                                                  '{} != {}'.format(json_dict['flow_id'], current_flow.flow_id))
+            semdict['flow_id'] = json_dict['flow_id']
+
+    if 'flow_event_name' in json_dict:
+        if json_dict['flow_event_name'] == 'end' or \
+           json_dict['flow_event_name'] == 'idle':
+            pass
+        elif json_dict['flow_event_name'] == 'new':
+            if lowest_flow_id_for_new_flow > current_flow.flow_id:
+                raise SemanticValidationException(current_flow,
+                                                  'JSON dictionary lowest flow id for new flow > current flow id: ' \
+                                                  '{} != {}'.format(lowest_flow_id_for_new_flow, current_flow.flow_id))
+            current_flow.flow_new_seen = True
+            if lowest_flow_id_for_new_flow == 0:
+                lowest_flow_id_for_new_flow = current_flow.flow_id
+        elif json_dict['flow_event_name'] == 'detected' or \
+             json_dict['flow_event_name'] == 'not-detected':
+            try:
+                if current_flow.flow_detection_finished is True:
+                    raise SemanticValidationException(current_flow,
+                                                      'Flow detection already finished, but detected/not-detected event received.')
+            except AttributeError:
+                pass
+            current_flow.flow_detection_finished = True
+
+    try:
+        if current_flow.flow_new_seen is True and lowest_flow_id_for_new_flow > current_flow.flow_id:
+            raise SemanticValidationException(current_flow, 'Lowest flow id for flow > current flow id: ' \
+                                              '{} > {}'.format(lowest_flow_id_for_new_flow, current_flow.flow_id))
+    except AttributeError:
+        pass
+
+    global_user_data.lines_processed += 1
+    if global_user_data.lines_processed % global_user_data.print_dot_every == 0:
+        sys.stdout.write('.')
+        sys.stdout.flush()
+    print_nmb_every = global_user_data.print_nmb_every + (len(str(global_user_data.lines_processed)) * global_user_data.print_dot_every)
+    if global_user_data.lines_processed % print_nmb_every == 0:
+        sys.stdout.write(str(global_user_data.lines_processed))
+        sys.stdout.flush()
+
+    return True
+
+if __name__ == '__main__':
+    argparser = nDPIsrvd.defaultArgumentParser()
+    argparser.add_argument('--strict', action='store_true', default=False, help='Require and validate a full nDPId application lifecycle.')
+    args = argparser.parse_args()
+    address = nDPIsrvd.validateAddress(args)
+
+    sys.stderr.write('Recv buffer size: {}\n'.format(nDPIsrvd.NETWORK_BUFFER_MAX_SIZE))
+    sys.stderr.write('Connecting to {} ..\n'.format(address[0]+':'+str(address[1]) if type(address) is tuple else address))
+
+    nsock = nDPIsrvdSocket()
+    nsock.connect(address)
+    stats = Stats()
+    try:
+        nsock.loop(onJsonLineRecvd, stats)
+    except nDPIsrvd.SocketConnectionBroken as err:
+        sys.stderr.write('\n{}\n'.format(err))
+    except KeyboardInterrupt:
+        print()
+
+    sys.stderr.write('\nEvent counter:\n' + stats.getEventCounterStr() + '\n')
+    if args.strict is True:
+        if stats.verifyEventCounter() is False:
+            sys.stderr.write('Event counter verification failed. (`--strict\')\n')
+            sys.exit(1)
diff --git a/nDPId.c b/nDPId.c
index daaac15fe..747ce0f3d 100644
--- a/nDPId.c
+++ b/nDPId.c
@@ -82,12 +82,10 @@ struct nDPId_flow_basic
     uint64_t last_seen;
 };
 
-struct nDPId_flow_skipped
-{
-    struct nDPId_flow_basic flow_basic;
-};
-
-struct nDPId_flow_info
+/*
+ * Information required for a full detection cycle.
+ */
+struct nDPId_flow_extended
 {
     struct nDPId_flow_basic flow_basic;
 
@@ -100,6 +98,19 @@ struct nDPId_flow_info
     uint64_t first_seen;
 
     unsigned long long int total_l4_data_len;
+};
+
+/*
+ * Structures related to certain flow states.
+ */
+struct nDPId_flow_skipped
+{
+    struct nDPId_flow_basic flow_basic;
+};
+
+struct nDPId_flow_info
+{
+    struct nDPId_flow_extended flow_extended;
 
     uint8_t detection_completed : 1;
     uint8_t reserved_00 : 7;
@@ -114,6 +125,11 @@ struct nDPId_flow_info
     struct ndpi_id_struct * ndpi_dst;
 };
 
+struct nDPId_flow_finished
+{
+    struct nDPId_flow_info flow_info;
+};
+
 struct nDPId_workflow
 {
     pcap_t * pcap_handle;
@@ -1048,8 +1064,22 @@ static void process_idle_flow(struct nDPId_reader_thread * const reader_thread,
         {
             case FT_UNKNOWN:
             case FT_SKIPPED:
+                break;
+
             case FT_FINISHED:
+            {
+                struct nDPId_flow_finished * const flow_finished = (struct nDPId_flow_finished *)flow_basic;
+
+                if (flow_basic->tcp_fin_rst_seen != 0)
+                {
+                    jsonize_flow_event(reader_thread, &flow_finished->flow_info, FLOW_EVENT_END);
+                }
+                else
+                {
+                    jsonize_flow_event(reader_thread, &flow_finished->flow_info, FLOW_EVENT_IDLE);
+                }
                 break;
+            }
 
             case FT_INFO:
             {
@@ -1135,32 +1165,32 @@ static void check_for_idle_flows(struct nDPId_reader_thread * const reader_threa
     }
 }
 
-static void jsonize_l3_l4(struct nDPId_workflow * const workflow, struct nDPId_flow_info const * const flow)
+static void jsonize_l3_l4(struct nDPId_workflow * const workflow, struct nDPId_flow_basic const * const flow_basic)
 {
     ndpi_serializer * const serializer = &workflow->ndpi_serializer;
     char src_name[48] = {};
     char dst_name[48] = {};
 
-    switch (flow->flow_basic.l3_type)
+    switch (flow_basic->l3_type)
     {
         case L3_IP:
             ndpi_serialize_string_string(serializer, "l3_proto", "ip4");
-            if (inet_ntop(AF_INET, &flow->flow_basic.src.v4.ip, src_name, sizeof(src_name)) == NULL)
+            if (inet_ntop(AF_INET, &flow_basic->src.v4.ip, src_name, sizeof(src_name)) == NULL)
             {
                 syslog(LOG_DAEMON | LOG_ERR, "Could not convert IPv4 source ip to string: %s", strerror(errno));
             }
-            if (inet_ntop(AF_INET, &flow->flow_basic.dst.v4.ip, dst_name, sizeof(dst_name)) == NULL)
+            if (inet_ntop(AF_INET, &flow_basic->dst.v4.ip, dst_name, sizeof(dst_name)) == NULL)
             {
                 syslog(LOG_DAEMON | LOG_ERR, "Could not convert IPv4 destination ip to string: %s", strerror(errno));
             }
             break;
         case L3_IP6:
             ndpi_serialize_string_string(serializer, "l3_proto", "ip6");
-            if (inet_ntop(AF_INET6, &flow->flow_basic.src.v6.ip[0], src_name, sizeof(src_name)) == NULL)
+            if (inet_ntop(AF_INET6, &flow_basic->src.v6.ip[0], src_name, sizeof(src_name)) == NULL)
             {
                 syslog(LOG_DAEMON | LOG_ERR, "Could not convert IPv6 source ip to string: %s", strerror(errno));
             }
-            if (inet_ntop(AF_INET6, &flow->flow_basic.dst.v6.ip[0], dst_name, sizeof(dst_name)) == NULL)
+            if (inet_ntop(AF_INET6, &flow_basic->dst.v6.ip[0], dst_name, sizeof(dst_name)) == NULL)
             {
                 syslog(LOG_DAEMON | LOG_ERR, "Could not convert IPv6 destination ip to string: %s", strerror(errno));
             }
@@ -1174,16 +1204,16 @@ static void jsonize_l3_l4(struct nDPId_workflow * const workflow, struct nDPId_f
 
     ndpi_serialize_string_string(serializer, "src_ip", src_name);
     ndpi_serialize_string_string(serializer, "dst_ip", dst_name);
-    if (flow->flow_basic.src_port)
+    if (flow_basic->src_port)
     {
-        ndpi_serialize_string_uint32(serializer, "src_port", flow->flow_basic.src_port);
+        ndpi_serialize_string_uint32(serializer, "src_port", flow_basic->src_port);
     }
-    if (flow->flow_basic.dst_port)
+    if (flow_basic->dst_port)
     {
-        ndpi_serialize_string_uint32(serializer, "dst_port", flow->flow_basic.dst_port);
+        ndpi_serialize_string_uint32(serializer, "dst_port", flow_basic->dst_port);
     }
 
-    switch (flow->flow_basic.l4_protocol)
+    switch (flow_basic->l4_protocol)
     {
         case IPPROTO_TCP:
             ndpi_serialize_string_string(serializer, "l4_proto", "tcp");
@@ -1198,7 +1228,7 @@ static void jsonize_l3_l4(struct nDPId_workflow * const workflow, struct nDPId_f
             ndpi_serialize_string_string(serializer, "l4_proto", "icmp6");
             break;
         default:
-            ndpi_serialize_string_uint32(serializer, "l4_proto", flow->flow_basic.l4_protocol);
+            ndpi_serialize_string_uint32(serializer, "l4_proto", flow_basic->l4_protocol);
             break;
     }
 }
@@ -1257,19 +1287,19 @@ static void jsonize_daemon(struct nDPId_reader_thread * const reader_thread, enu
     serialize_and_send(reader_thread);
 }
 
-static void jsonize_flow(struct nDPId_workflow * const workflow, struct nDPId_flow_info const * const flow)
+static void jsonize_flow(struct nDPId_workflow * const workflow, struct nDPId_flow_extended const * const flow_ext)
 {
-    ndpi_serialize_string_uint32(&workflow->ndpi_serializer, "flow_id", flow->flow_id);
-    ndpi_serialize_string_uint64(&workflow->ndpi_serializer, "flow_packet_id", flow->packets_processed);
-    ndpi_serialize_string_uint64(&workflow->ndpi_serializer, "flow_first_seen", flow->first_seen);
-    ndpi_serialize_string_uint64(&workflow->ndpi_serializer, "flow_last_seen", flow->flow_basic.last_seen);
-    ndpi_serialize_string_uint64(&workflow->ndpi_serializer, "flow_tot_l4_data_len", flow->total_l4_data_len);
-    ndpi_serialize_string_uint64(&workflow->ndpi_serializer, "flow_min_l4_data_len", flow->min_l4_data_len);
-    ndpi_serialize_string_uint64(&workflow->ndpi_serializer, "flow_max_l4_data_len", flow->max_l4_data_len);
+    ndpi_serialize_string_uint32(&workflow->ndpi_serializer, "flow_id", flow_ext->flow_id);
+    ndpi_serialize_string_uint64(&workflow->ndpi_serializer, "flow_packet_id", flow_ext->packets_processed);
+    ndpi_serialize_string_uint64(&workflow->ndpi_serializer, "flow_first_seen", flow_ext->first_seen);
+    ndpi_serialize_string_uint64(&workflow->ndpi_serializer, "flow_last_seen", flow_ext->flow_basic.last_seen);
+    ndpi_serialize_string_uint64(&workflow->ndpi_serializer, "flow_tot_l4_data_len", flow_ext->total_l4_data_len);
+    ndpi_serialize_string_uint64(&workflow->ndpi_serializer, "flow_min_l4_data_len", flow_ext->min_l4_data_len);
+    ndpi_serialize_string_uint64(&workflow->ndpi_serializer, "flow_max_l4_data_len", flow_ext->max_l4_data_len);
     ndpi_serialize_string_uint64(&workflow->ndpi_serializer,
                                  "flow_avg_l4_data_len",
-                                 (flow->packets_processed > 0 ? flow->total_l4_data_len / flow->packets_processed : 0));
-    ndpi_serialize_string_uint32(&workflow->ndpi_serializer, "midstream", flow->flow_basic.tcp_is_midstream_flow);
+                                 (flow_ext->packets_processed > 0 ? flow_ext->total_l4_data_len / flow_ext->packets_processed : 0));
+    ndpi_serialize_string_uint32(&workflow->ndpi_serializer, "midstream", flow_ext->flow_basic.tcp_is_midstream_flow);
 }
 
 static int connect_to_json_socket(struct nDPId_reader_thread * const reader_thread)
@@ -1512,7 +1542,7 @@ static void jsonize_packet_event(struct nDPId_reader_thread * const reader_threa
                                  uint16_t pkt_l3_offset,
                                  uint16_t pkt_l4_offset,
                                  uint16_t pkt_l4_len,
-                                 struct nDPId_flow_info const * const flow,
+                                 struct nDPId_flow_extended const * const flow_ext,
                                  enum packet_event event)
 {
     struct nDPId_workflow * const workflow = reader_thread->workflow;
@@ -1520,7 +1550,7 @@ static void jsonize_packet_event(struct nDPId_reader_thread * const reader_threa
 
     if (event == PACKET_EVENT_PAYLOAD_FLOW)
     {
-        if (flow == NULL)
+        if (flow_ext == NULL)
         {
             syslog(LOG_DAEMON | LOG_ERR,
                    "[%8llu, %d] BUG: got a PACKET_EVENT_PAYLOAD_FLOW with a flow pointer equals NULL",
@@ -1528,12 +1558,12 @@ static void jsonize_packet_event(struct nDPId_reader_thread * const reader_threa
                    reader_thread->array_index);
             return;
         }
-        if (flow->packets_processed > nDPId_options.max_packets_per_flow_to_send)
+        if (flow_ext->packets_processed > nDPId_options.max_packets_per_flow_to_send)
         {
             return;
         }
-        ndpi_serialize_string_uint32(&workflow->ndpi_serializer, "flow_id", flow->flow_id);
-        ndpi_serialize_string_uint64(&workflow->ndpi_serializer, "flow_packet_id", flow->packets_processed);
+        ndpi_serialize_string_uint32(&workflow->ndpi_serializer, "flow_id", flow_ext->flow_id);
+        ndpi_serialize_string_uint64(&workflow->ndpi_serializer, "flow_packet_id", flow_ext->packets_processed);
     }
 
     ndpi_serialize_string_int32(&workflow->ndpi_serializer, "packet_event_id", event);
@@ -1591,8 +1621,8 @@ static void jsonize_flow_event(struct nDPId_reader_thread * const reader_thread,
         ndpi_serialize_string_string(&workflow->ndpi_serializer, ev, flow_event_name_table[FLOW_EVENT_INVALID]);
     }
     jsonize_basic(reader_thread);
-    jsonize_flow(workflow, flow);
-    jsonize_l3_l4(workflow, flow);
+    jsonize_flow(workflow, &flow->flow_extended);
+    jsonize_l3_l4(workflow, &flow->flow_extended.flow_basic);
 
     switch (event)
     {
@@ -1619,7 +1649,7 @@ static void jsonize_flow_event(struct nDPId_reader_thread * const reader_thread,
                 syslog(LOG_DAEMON | LOG_ERR,
                        "[%8llu, %4u] ndpi_dpi2json failed for not-detected/guessed flow",
                        workflow->packets_captured,
-                       flow->flow_id);
+                       flow->flow_extended.flow_id);
             }
             break;
 
@@ -1633,7 +1663,7 @@ static void jsonize_flow_event(struct nDPId_reader_thread * const reader_thread,
                 syslog(LOG_DAEMON | LOG_ERR,
                        "[%8llu, %4u] ndpi_dpi2json failed for detected/detection-update flow",
                        workflow->packets_captured,
-                       flow->flow_id);
+                       flow->flow_extended.flow_id);
             }
             break;
     }
@@ -1998,7 +2028,7 @@ static struct nDPId_flow_basic * add_new_flow(struct nDPId_workflow * const work
     switch (type)
     {
         case FT_UNKNOWN:
-        case FT_FINISHED:
+        case FT_FINISHED: // do not allocate something for FT_FINISHED as we are re-using memory allocated by FT_INFO
             return NULL;
 
         case FT_SKIPPED:
@@ -2406,7 +2436,7 @@ static void ndpi_process_packet(uint8_t * const args,
 
         workflow->cur_active_flows++;
         workflow->total_active_flows++;
-        flow_to_process->flow_id = __sync_fetch_and_add(&global_flow_id, 1);
+        flow_to_process->flow_extended.flow_id = __sync_fetch_and_add(&global_flow_id, 1);
 
         if (alloc_ndpi_structs(flow_to_process) != 0)
         {
@@ -2463,25 +2493,25 @@ static void ndpi_process_packet(uint8_t * const args,
         }
     }
 
-    flow_to_process->packets_processed++;
-    flow_to_process->total_l4_data_len += l4_len;
-    if (flow_to_process->first_seen == 0)
+    flow_to_process->flow_extended.packets_processed++;
+    flow_to_process->flow_extended.total_l4_data_len += l4_len;
+    if (flow_to_process->flow_extended.first_seen == 0)
     {
-        flow_to_process->first_seen = time_ms;
+        flow_to_process->flow_extended.first_seen = time_ms;
     }
-    if (l4_len > flow_to_process->max_l4_data_len)
+    if (l4_len > flow_to_process->flow_extended.max_l4_data_len)
     {
-        flow_to_process->max_l4_data_len = l4_len;
+        flow_to_process->flow_extended.max_l4_data_len = l4_len;
     }
-    if (l4_len < flow_to_process->min_l4_data_len)
+    if (l4_len < flow_to_process->flow_extended.min_l4_data_len)
     {
-        flow_to_process->min_l4_data_len = l4_len;
+        flow_to_process->flow_extended.min_l4_data_len = l4_len;
     }
 
     if (is_new_flow != 0)
     {
-        flow_to_process->max_l4_data_len = l4_len;
-        flow_to_process->min_l4_data_len = l4_len;
+        flow_to_process->flow_extended.max_l4_data_len = l4_len;
+        flow_to_process->flow_extended.min_l4_data_len = l4_len;
         jsonize_flow_event(reader_thread, flow_to_process, FLOW_EVENT_NEW);
     }
 
@@ -2492,14 +2522,14 @@ static void ndpi_process_packet(uint8_t * const args,
                          ip_offset,
                          (l4_ptr - packet),
                          l4_len,
-                         flow_to_process,
+                         &flow_to_process->flow_extended,
                          PACKET_EVENT_PAYLOAD_FLOW);
 
     if (flow_to_process->ndpi_flow->num_processed_pkts == nDPId_options.max_packets_per_flow_to_process - 1)
     {
         if (flow_to_process->detection_completed != 0)
         {
-            jsonize_flow_event(reader_thread, flow_to_process, FLOW_EVENT_DETECTED);
+            jsonize_flow_event(reader_thread, flow_to_process, FLOW_EVENT_DETECTION_UPDATE);
         }
         else
         {
@@ -2547,7 +2577,7 @@ static void ndpi_process_packet(uint8_t * const args,
     if (flow_to_process->ndpi_flow->num_processed_pkts == nDPId_options.max_packets_per_flow_to_process)
     {
         free_ndpi_structs(flow_to_process);
-        flow_to_process->flow_basic.type = FT_FINISHED;
+        flow_to_process->flow_extended.flow_basic.type = FT_FINISHED;
     }
 }
 
diff --git a/test/results/README.txt.out b/test/results/README.txt.out
deleted file mode 100644
index e69de29bb..000000000
--- a/test/results/README.txt.out
+++ /dev/null
diff --git a/test/results/anydesk-2.pcap.out b/test/results/anydesk-2.pcap.out
index 995bebe2a..bba7b6c48 100644
--- a/test/results/anydesk-2.pcap.out
+++ b/test/results/anydesk-2.pcap.out
@@ -149,7 +149,7 @@
 00177{"basic_event_id":8,"basic_event_name":"nDPI IPv4\/L4 payload detection failed","thread_id":0,"packet_id":240,"source":"anydesk-2.pcap","alias":"nDPId-test","l4_data_len":4988}
 04027{"packet_event_id":1,"packet_event_name":"packet","thread_id":0,"packet_id":263,"source":"anydesk-2.pcap","alias":"nDPId-test","pkt_oversize":false,"pkt_ts_sec":1613977603,"pkt_ts_usec":313834,"pkt_caplen":2745,"pkt_type":2048,"pkt_l3_offset":14,"pkt_l4_offset":0,"pkt_len":2745,"pkt_l4_len":0,"pkt":"KDc3AG3I2MuK4S0uCABFAAAAd0FAAIAGAADAqAG7wKgBstOUG56PHFwWWcP+5lAYA\/2ExAAAFwMDCn5Z4hNO+msU3mQde8XauUXuibx23ZWOy1lfKlJdZeS7jbArOCdma5G1Oyj\/YHi4OgowtlsF3GCyvWAIcIaf2qvcUvo8xw+u+vFmA88\/ZFtsQrOzjtOQOjqB1w9IVRQDv2grZ\/g+TPfVtI2fmulJ6\/DmyG6skD8\/NJUWgh8nP2od5WwvpSZMAK4lfKeb2z0dUsUOya9f5mvFpFHQtfrBKUxJ+Qi3RehBwzTb55Ty8p+hZJJV7Iwzjy4pVLDEN902w5s1zPRfh8wp6Anmbv2bXEn0A43qLMRFkgBP5BI+igCLV7CUZs1YGzWqE2Qe5LZSZvQs9XJzBdK9uD29IREqU0HJNfHTRP5llSd7+z4nImKHOyeI7anpkMzlguOkoVfJQEajpvI3WE9X\/1CUkRjvlueAQCmBNgaGjBqVDukdBpXNlB3n+105fzXVNBRay4p\/b\/GcjzrAXO3IE9M5nF02cdlxykTYZ\/v51qwrR+BW7MUkGi3iYyceDRneOdImiEA8Fx2ms52erPvVB5WPOPh9wK4Bx+Olb78ikN5Z6ABTeAlOOHzSwfkuoLGi3VXydpd80btVVx94fLuzAnSflm6lw5yfcuyRzOr5GQAuVNsGcgID\/tfJAgJBy5t\/3GV7d0R70TiSGpc0dA8ovueO+whcxGg6XkvNylfFnAymmw0H4NMNkLqqo252fRfF7\/bzwTJQowsRZQOWGBqNMHe+7deiTVbvVwZYIsdMXNebCII3WU2oVVD64POtpmYsGKOTF2T3fYzkHHLWAVWGQXU1SehD\/X6lH4iv0uzHEKfY\/Hw8F02O8iWssDHjxDLKVUsInHxGwWBZgbF1MU72FDnuHR5CGNKs19Jbx9I2Kk7XDMbfxxqgpUygmTtFPYFKryt93oYMUkjkspSKsTBRkCWXQuaQo7qu35UlkH2lUKiV09U1wYPedcqUsQ92UbGj\/siMqOIyeQowgB+tEpc75tZfM8xnZmaiFsP4Vbf7x2c\/9r5dJp0GY03Yhup7L6msnnDvEn666l\/wb26yt\/yCGM\/WN68jMfQ9IAH+C39Dcs3b\/+kwvAnD044ZM3+CUM8hQFmGwe94aPz0bI46AuTKmNNXtxdN\/UJWxOhk3Slo\/7+xVgIu6ryQ\/3gqxm0qSPUTi4uLVp6WeJiJEXZ2OYpVb9Fy8UCEez\/wS41UwuJPv9fT\/EMbWyowl7srODAru\/H73XdW41KrMalzeWf6Mnb80av5KwiOs2Y23EoAu4D5z21i4Djf9v9ODq2KUOHe9qEvjwxVEnt1qjhsgG+OjPvdbTT6\/9Ya7HaguBU9fuN3skEP7nGJfAq5gvs9hwzjCnB2a4GfmzDhmVfHwtgGTFvvXET24NHuZ4K\/8PXaQD8fBsQPzmNoslsonoxEPXlubw07HA7kKD+zNBa6FR1oTEAvBYYHKVjVMGlbNwITm1Qe+SWAuqxnY1eq541bN2ZEe9inXHIZnCVkpt9QFo2+Wnlii6gpZKNvdvJGlt\/Ck9K\/d3yfuDmJ2HoqpJzzoojRioHe9nS6KdtHQiVxWDCHyTmPDoeJjFgmNShc1KNJCxdYSrbkpXIAJvp+2EtxPnijllODqp9E1tFwH\/rzveFmx+Wc1K7P3nLChjoT3ufyQk2mhbp93u\/64NyqZVuH7fRyBlfDOR8yN+BEsixebRyiiK\/FnZJ5fLjfhgVme8+WX021lqeGUdX3m\/VkkyJXsLdoBOPanm+WsGtt6san0iXRmZTigkrHoUlUqrF+qmPvvGm4dgD5dKZXTfVVcTvCeBoWiu84Jakxdh0f5VPyQtD5ET57bn8KGcxpAXRxzH6jCiH4XJoOqxeENkjlNoX\/E9R6S5uAACvvrA+ORK8fhz5MVGKF957Ut5GZNW84r\/Ky2TYqrF46WgAZBGJux69\/T4D1US4ZkgNfUGfpuRGDdidMFNf6yW+ITJBzigOL5NJlsMkQOChbTmqlMe3ls+Sb9u2RcrE33nNSiQxahx1SH2r4CGe4a7tQFwvlhdpZphEQzqrbUvlU3xdCMtTxxne3XgGSF8j88eoUPM0jqDUPlrBbvd5mXogZYZLOEfpyiMSNnbwIvEq3R6cCrmh5DdIorOBdj+RAuOyzSD4Z\/2iae2GDHelQyAemjxPKnVE2d0KVuxBWFPtd1zdWXTDCyFU5H5lDpkgf1mzHiNrvqSpBI7YVjs7mwDQOgbo9RmT4uxCkEDz5IWg47a+P4f5fcyxrdxQTjDQIpN7uN2CBc2Oq7JDZQYuUrbkmwr6hIAG0JW31HQHIKbQw56Eq\/UwylTe5Yw3Xapi+ctffH8Fjo74SgVjOfurQWtPkJ9y1\/XCYJlkQj4Eq9NGKVml798jEO3kWgIeLF2jcL\/xBEFbrjd03vbXKYB1444cMPq+N0eZDETbjBDQsHeHxvCVoTSxbrakgRAQrc3H+aBqBNYRqoVwtvSSFd8iLiG8W+DEr5zp94CSESrQl6Z1\/VXyEAlkGYB4NLUO\/vDEyyviQJyNtmzhFLw76uw+al1LSas8zYAYzxy9kQ4rSDMZ\/wy\/xerQQN8zFOZZ8My4SWoU+5ig+EmAZjEK5XhiKMyvL9KoFQqLei0e8SmBe3lYb\/th5YG37aWDNvw7KvJZCtvAGp55TDsGVsNU7Fcv69v3YfLy66ZJ09UzqvnzQOTuNkBcMftHM1AvQ7FLM5FN49i089r0\/PDCaCegLQIa8yH4jZrCgiAK9DWPBsJOYCcVcnTyElMFGWKAQDuy1ySm9g6fXErhjvXhHTS+t9a7UxzxKaObgCXnBVCEULXe03mmu6RWF8GioBeesdzkyfjBjHk13FB+ujRnul6P\/dcW7e44Iw1Sx6zdRz6QcbuAdMXxeHZ4bm6MuTvlVw85lnaquFyRVNxzYZfjSsR8b3Ny2hF370r70\/0L1mFO4BBnD503vyP5FGEUer6jOORAbVTvjkfv7DfT5ce+mqBnd7hI9nyQza5Z0fatgMDGKwWiclhCNav+XhjFgM+Mwr14C2gJjUDg9mfO52JQBrmzyQuDTC1bfYod7Vodp\/oStGrztMdFIBGm4gqba7qS0CZ7u9eU+lY6j57OMtLpGXhbzy6fEEUkWLB9\/J6wcBps4b9P2obOHVJ45sa+as0LsL1RcdUCU8bHEUzFkgHWDh5Bx6gLVQQmPtT0+kXpnPw8VH7nAt5zbP9PKg9mkYMdlrpXIZQYH\/vYZ\/s4\/AO+h5uy5L+gjfhFfEim+1bTLMvy\/gIapPFI+FVw78Eb39bDVBsZhXArGP72zkjqH60HyLuuVPZr6X+LiRvTF4ct4kmA\/t3Q8QPnOFyiRxqDR82tP4\/aMS0FPR4Sq9rD63\/BKuBBcmRXTwRi82ovnDQdBp35qpuj9GdbPJjSQE2nfmX2hsX6Xk76ZHbaL8KjLyiEkhDJl4ImfOLo1YPuIq1a3DUWjFYRw8EY9o0UkUO568j\/Fc\/yC\/CfR4bTRmkKaj8Hr4ucVe2POT1Wd1gY+y2vQppzcKvXvnmHhNabFyqyW99JpzheV2QazE\/pof2oLvgPRXNjBs9DyMCTvOSAAhuyUC+3+iJ4y7VqFLJJ88sglwH+eYe7d5DWImyW5UB4S"}
 00177{"basic_event_id":8,"basic_event_name":"nDPI IPv4\/L4 payload detection failed","thread_id":0,"packet_id":263,"source":"anydesk-2.pcap","alias":"nDPId-test","l4_data_len":2711}
-00962{"flow_event_id":5,"flow_event_name":"detected","thread_id":0,"packet_id":326,"source":"anydesk-2.pcap","alias":"nDPId-test","flow_id":3,"flow_packet_id":255,"flow_first_seen":1613977595379,"flow_last_seen":1613977604238,"flow_tot_l4_data_len":21934,"flow_min_l4_data_len":20,"flow_max_l4_data_len":1480,"flow_avg_l4_data_len":86,"midstream":0,"l3_proto":"ip4","src_ip":"192.168.1.187","dst_ip":"192.168.1.178","src_port":54164,"dst_port":7070,"l4_proto":"tcp","ndpi": {"flow_risk": {"15":"TLS (probably) not carrying HTTPS","24":"SNI TLS extension was missing"},"proto":"TLS.AnyDesk","breed":"Acceptable","category":"Web"},"tls": {"version":"TLSv1.2","client_requested_server_name":"","ja3":"3f2fba0262b1a22b739126dfb2fe7a7d","ja3s":"ee644a8a34c434abca4b737ec1d9efad","unsafe_cipher":0,"cipher":"TLS_DHE_RSA_WITH_AES_256_GCM_SHA384","issuerDN":"CN=AnyDesk Client, CN=AnyDesk Client","fingerprint":"F8:4E:27:4E:F9:33:35:2F:1A:69:71:D5:02:6B:B8:72:EF:B7:BA:B0"}}
+00970{"flow_event_id":6,"flow_event_name":"detection-update","thread_id":0,"packet_id":326,"source":"anydesk-2.pcap","alias":"nDPId-test","flow_id":3,"flow_packet_id":255,"flow_first_seen":1613977595379,"flow_last_seen":1613977604238,"flow_tot_l4_data_len":21934,"flow_min_l4_data_len":20,"flow_max_l4_data_len":1480,"flow_avg_l4_data_len":86,"midstream":0,"l3_proto":"ip4","src_ip":"192.168.1.187","dst_ip":"192.168.1.178","src_port":54164,"dst_port":7070,"l4_proto":"tcp","ndpi": {"flow_risk": {"15":"TLS (probably) not carrying HTTPS","24":"SNI TLS extension was missing"},"proto":"TLS.AnyDesk","breed":"Acceptable","category":"Web"},"tls": {"version":"TLSv1.2","client_requested_server_name":"","ja3":"3f2fba0262b1a22b739126dfb2fe7a7d","ja3s":"ee644a8a34c434abca4b737ec1d9efad","unsafe_cipher":0,"cipher":"TLS_DHE_RSA_WITH_AES_256_GCM_SHA384","issuerDN":"CN=AnyDesk Client, CN=AnyDesk Client","fingerprint":"F8:4E:27:4E:F9:33:35:2F:1A:69:71:D5:02:6B:B8:72:EF:B7:BA:B0"}}
 03262{"packet_event_id":1,"packet_event_name":"packet","thread_id":0,"packet_id":359,"source":"anydesk-2.pcap","alias":"nDPId-test","pkt_oversize":false,"pkt_ts_sec":1613977604,"pkt_ts_usec":476233,"pkt_caplen":2184,"pkt_type":2048,"pkt_l3_offset":14,"pkt_l4_offset":0,"pkt_len":2184,"pkt_l4_len":0,"pkt":"KDc3AG3I2MuK4S0uCABFAAAAd6tAAIAGAADAqAG7wKgBstOUG56PHIiRWcQFglAYIBOExAAAFwMDCE1Z4hNO+msU9LBvnBdfz8pT3uIpAXl9v8baPCclctzafizqusRLc+3yBRrxsTQWeM+z\/j15TWKILyUHSn+85MmEVgMVvQ0naJDIPu9CFBTDGola9mExfWT+oniDrqVp1gDABVnjk7XDV+j312n\/hzyqb4ibRnC+bFrWzgCW1GEKZC1q\/E\/6hCR8a6NWoWAXlJURq1D\/2FNJECXg84tGVTlZUZ1hjKYFe1ajrioO0kHG42Cd8Zjh8z0Xueajz3JAzS640hLUA9UiOwKymZvLlEbzmhvESjy7FaJ9bekboPw3bn\/Jlj8ZF47zmQeEehl2qQ6htreM+LkT10pyuawnjdSA49JLH62hAyXThdYpAqJip7Of2\/W4b1J\/sOcFmX3l9KAFnOoqthb7U+hWo4LNbMBAreRWvbyJBqBBBkZtLMF3OI\/lgS2KRgiGqWPlc5\/8IqmFk9teB0eXT3W90Ps0UUvmRSCUsuyjlE2EUCed5yhGbXvuJ8xSirr7nIFa1dYweQN7QjZ0sg00UI7aXkhkgieHniYh7BkAzTo5ugnGnsZrDAocUbXzyptfnLrllkciPWt6N4rg8c\/xwdNBoEXRr4P2mFanIOSfwLfesF\/8nIMB4jD4dzmMHqwwnijrCTzHMjJCWGjBiVcn\/UQYAqSGPRdj\/olBVabavlmrH9Royswmu55\/v0PdSgyGh\/aF1NdAgMCQWfK4iTLXOnXiEhxUmiaGmwJhqej+pUp9yjksckOwAldsZVm2TEe1KH7VnJBozetryh05+IDLSkv0zfcXCynFCbOfRrXJi9E5rMp+EFcmkCd5du5qCwA7mioeIjdmsg6o\/hZay9NNqv+SjegBeEnWjGidm62Bg2J3ugleU05MdTEjPG\/0WEVFE4YLoZ0+Rmk27LsJ83E69N6EM7LIqHaBy4YgdBXCwRYXMBiZ7\/eXyR8ouKpqBrgyc0zmgTMfEguyU7bGFL8oz\/66InO2PDAb3K1g9EivYV0J8FGZbXrGGgeE23xb1i3E7zCa4xispnmUp8ZnvfmRqlLtxCp9xRo6wVZs\/8OOR3ozRmiI\/PMUf2ocLk1A7EQ06Bysnei2m9sDgUmz3xW18h43AuI3Dq2dw8luofIYO2mIw8PGK3r5t2XcHhzApuS2sJNMJzPVZjPnXGXlhTtPZq8RtPkaHlZqnY8opMkhjFF9Aqz3\/NEmWCFimFinDFcmhKzw4Zc11XVddg6SqbuK6go4CDvysm0p0t9NPekVu4zDVD4EAMuugSYVQPLC+GjcaxjX9UJufqiIKF2iGtSmbJ5\/R0oXR49FUnI9yHKXJ4k1LJbs5ulkD\/zGTnwCq17x21cHuxnM6jXwS\/ZjHHSGC3ISErC25VTJIcskqau\/dLYahxzXBtlEISVUbywuDuTbM8bVfs1bmyjIqYpqoABDoN8znMk4tsz9h\/kXlXKkCe6C+ec5cX0UVZQMIW14dtHVYELwX+yQ11ENYgNnbDvK9eYwU0VtThgC3i1tU+NwupUlXjxfhWt4d9x+S1Drfg2\/F29sYlDkdvYZFNRxoce3hBgJMPkIZEwqQFdENALY7ybsrObH42iP1NFKqM2PiLlHgVkrXHPep5p5nTaEGT1K4XQKFidsDE\/TU5jp+uV5i7tmWslQ3X0hd1lqhRPKzFSxhrdL\/OkYNUrKk8pswZhw3Z5L1hzdrsD27Qhrf+B3glSilptp8X7Eb52KYHrcuXisGa5DME2Lrzq7wHZEcCZuFh\/f9pqqJYEw3qNzgBQZCUbbeWgAPqTdMSOTev3F1ZSLvjeDledsYbcWvH+19SSbYW5Y+wa3pdx8cHj9rgNJObLJ0gF\/YxIeWBWbMgRPm9VI884Bq0CmrSk7ddVJJwqxpMhp3yO6unpbvR+zfTdO\/gFuftha41xabyjq2RbwbJS\/QEAhDCTueFRp8UI79s9E8eeZNx9EvY6Nti3XxVxAo3tbUi6gx1ha8BjET5MrziHMVJP584CS0eGAzo8fj1U9Uc+O6iOZvqO0xkwHZXp+13zpS+c+REzva4Oj9b6ImTr0r\/rqGg9rLH+ngtAU8Go4I7MCxaT+qMw3Sn\/jD1ZwNCOtlEOXIH0ppz6oLuqXGCJt0v8B4q3O9\/iS4Etdlwc5FwSC7vNZeM7RhhTBOd920Cgdf6+edNDGmsNO4htWQFAC0nm4yH7hY\/lMyTQ\/Go58thZciiFw4Cej0V0w9z1lZr0Y19WT5BpU\/41Rhs5jiD4sEvnn5fsC0k7V8yO3RdbF3LAesZcbukPjgqMXj48hBw8gAwlDe4wqdAR8FzU4xAgi67KDy5J9aahTmpodMn3eAlq2seT1sprowIc5H2Jr6vfv0RSDSBv125+qvt0xa5w4kAcrHbM+eOH0yjmMG3GLfBJVMa4Vk1NsKaJ0UQ+RHQJfAUAyJ8xY4LRIPsJajoH2jPGjFbI4LDI8bhoRIdBFUKHN9uZjbq3H5dTZloX6t\/+mVMaOBCiuB0wF96KeaIfPnoIAfsOIL4RAjJpyEA8YqwiLIYneZIciytK4JU0djusymFsgD3QmBLLM3T8wmJfmQs+XxdV6LUZCGbP48aNe2PEu4cgNFp0Gedax29OKBqKQJrrDAOojGxNEFqD+wgFm25xNUI\/oXWJUXCHAhyWvKF06pmsW8PgL9krA7cX3OGZh+fx6Ouf09uuPaEUfCe9q0DYD5wRHLyGMQuCzEVKuvUYxbp4bFbcuyJYIyTf6WEilDAJELMx+kjzm\/H5Jsd3GEHZoFCHlCgDalTY8TAlsEpBNykvZp6\/PHoKQjUrmjAolT9SDrLsJqIlaBNF2AmQ\/Iyl1mM2T2GFnQmg84apLYPcFrVeD"}
 00177{"basic_event_id":8,"basic_event_name":"nDPI IPv4\/L4 payload detection failed","thread_id":0,"packet_id":359,"source":"anydesk-2.pcap","alias":"nDPId-test","l4_data_len":2150}
 02586{"packet_event_id":1,"packet_event_name":"packet","thread_id":0,"packet_id":429,"source":"anydesk-2.pcap","alias":"nDPId-test","pkt_oversize":false,"pkt_ts_sec":1613977605,"pkt_ts_usec":157936,"pkt_caplen":1685,"pkt_type":2048,"pkt_l3_offset":14,"pkt_l4_offset":0,"pkt_len":1685,"pkt_l4_len":0,"pkt":"KDc3AG3I2MuK4S0uCABFAAAAd+ZAAIAGAADAqAG7wKgBstOUG56PHJqJWcQKplAYIA6ExAAAFwMDBlpZ4hNO+msVBGAU\/CQ+++T94X6aPp5XOKoWc8p1LFmHdGVPh9BmLi6fwPmaM4TfFP18K97w+JrG2mMRmj1tdgTVpDxt1C0Gncnny4rfOTrXTSleZ5fZVTWiCSG7aNbBqkDSL349Eg9z8IeGPlnYoPEN4tP0hVZemLcZgvILCgm49DsuVR3nhYp4sy6rhIgg2ckXEUXDokzjgL1yjIvt0ScqIB4okJR1wK79N4XZqDHUn2McD1b0N9v3pDlMk30O+IeVoz9StvBgQxoSM1A5v2XOynHvcw3I8aid5vEAfPOQwi7MSG1PCJ0p3e78RTR5AvwoWV5tJbAp5WWCvBGG8HYJ6RuplivNDXK02J7ld0qN7u7Q\/nmAnAOYa\/GWwRKg9Tr4zfIcTQXCCMH8YRxab5gJYESXf\/z1ewgfmFdNttFpDtF3N7hOkJJmZHsJzuVof1rADifgRt97Zt+Isn2GstbeNF7UKJMLnv75OfDd2jVaGyCWOSqr\/89o5b0Qcba9pNbd27IaXMZ396LcYhHzQDlZLBOMY+gl3DT40bd0Qn3wMvCOe79J\/29yZ6+yHg0PB8z38SVANS+MLgd5MHawzoK6qP\/KoynzQmsUhdMqkAc0u5QRyWPT6U3NnyyEfroJ1LxXZiO0p\/fJaarHw0cLP1fjQ7KcB\/LPZEOL57GO\/hkiUjKlr9T\/zgfe0MpybuxtbUS5tZFJNfvjqzwCDWxHE6QvgtJYBEYICQ9457KYO\/wcbNLey4CBV4x\/6U3oxvEnGBaUwLbpibk59xCzCzXuzLKOU2h\/EHV6JrEnWQFj7q+IE54AaAPZmfjFNLhs8FI3pZolVNQe96OFf7k5LQqyxz6oZ1rNO+dd9\/S1xOcgbh4tB3VzpRlIif9Xfi6vpxQgpAp\/Ckg4g9P2rmweTngy7EZcRiPY\/bi2lc5tqtIT2YjwokS+09PlxQOwAQPW9v9MUl+HVmH9C+i2v5UfxK\/4ypGKOP4BxiKQOzzNuz++qNx\/SX2yG+XNVmn4xGXdzlc2H8mwNwrvpt0+QLWBHW19hkrtqlSNDdhPAKKnjAc9OMKU7xejzXqyMWinIDTpLMEj5I3dKLCyxRQJXLZTienT2QWOT\/xdc50wNc4XYcA+6WBY7IyfdBJBT+rLgTGDSH1\/zTXELobM+rGuJkzTFRw9bqFFXtxSCc772VJ12sjK0vXDvFWKoFaNoe78LZ5voDtMwqopYvwpV7H6nPpWna\/o4CSRCyA3G14Am\/fxios0att5z9q+drHHVURelxPIt6ukJOio91iJVDLpBHbf1hgwox0kd\/+SeiP1mSjU2kGz8LrctjvSpmSRN6a6sKEorwbTCfZd78Qn2UaEncdDQIPr3BaGwPF4TGFI0Wu\/hgVJlFDzcuBsXN4DnS0YuWlgUdm0mq5mHA6s6lEm9Sw10GlrxnAmjH85PGF8NK+bJAyFRbkgKNmxeLMD2\/fJM9Yy30wqYmAchsBRZiFltsLa0nUe+XTAR9Hq2HXsEEZ4EdZwmwTjJRctTrzyhro2HYoydJS1pGm0+nd1efNqtke4yktOnOtU1KavI+p+2vrcYUysE5QjNXan78ayVsfgNcFNqMFZS8HNwDAfprS4urmn6HN0VMtMdjgGQRPG16qegP966dnrBVAaVqv7RxbWSqR9ZgtQN4kznoApYsQ\/htBNdcpggCk7aEeCp4hqA5E3Dgh9f+uZnbb36LAJBjvyFcmH81G3Lk5YlhF\/zSVvLKUb0MTqYenR0yMx4zxUl4GHoYotJkkka9m13vFT0upUDpqUYIAPW\/ssc5jgoqrMk9Hhi5y+7HWKQgjrdb6nOU4S5uyGOKDW4mE6\/rPBHp0fY5ylYs03GpUua9a\/glfenyNSCemqQlPjbCThLJwe2Q+jRt0ZttjgtfXYUtXdKQPdi9kDvDLF6bC6lPNdETt6RPwULQMHokPt4D2I843jsNIop+cnms6WRAoTEy\/nJlP0Xf6+O2AOve7kIGWj79Hb+Txxi7fe6XvRzr\/AFPz42M5rbE1CpgUgXEzV9+mwpu53B6ibIPrxe165c8h5iqFjNOd91m2C48D0xK3n27tv8SErJpnkzhizKwvbaMs382VOUnMh31zjjLabE7N9jb\/tEo1n8oAFoazbbRyR7uBihqWES0IVOQF2l2EDE0lPDLdJyXw="}
@@ -923,6 +923,7 @@
 03252{"packet_event_id":1,"packet_event_name":"packet","thread_id":0,"packet_id":2513,"source":"anydesk-2.pcap","alias":"nDPId-test","pkt_oversize":false,"pkt_ts_sec":1613977618,"pkt_ts_usec":195735,"pkt_caplen":2180,"pkt_type":2048,"pkt_l3_offset":14,"pkt_l4_offset":0,"pkt_len":2180,"pkt_l4_len":0,"pkt":"KDc3AG3I2MuK4S0uCABFAAAAiTlAAIAGAADAqAG7wKgBstOUG56PNqDiWcRUSlAYIBKExAAAFwMDCElZ4hNO+msXh2bUnODOizPQiAddzId5s0L6Tw4B2tyJ0YG6Lk9xeCjCkk5iyV\/TcZ+MkCeyOMxeyKngR\/0L81CbJvcp2HY5+PJ99RjORMM0hFqL0M9FsClysAMVqzfPzL0uTI+AMuQYjmp2qqS3n0jD5Mc\/OI6AQf5alcy5blcc+SRlMpLpNTjaoZqhbqyN2OjZAQ0RCghY1jgDpcpPpjzXwFfM+eUtNLtomqzUozUMvSGBCPubR8ysHpKf08rz9nQsYe\/eQy1W+fGZ3UevRU4e0ziP\/Z4ImlVCjTNEZ4Q1m5e1dfxc\/2iPO\/xRUkfR9tTq5C7ck1L6BG5Sbs7srBImqkQCfZO0borStlxpNfdnOV3FAeKPjPu+OB0GQFdSoxU3ShgSCS+s3yhVPiImHbrFfcRtfPcymodIF1QSeUI\/b4QvFBs1xUsetwKnOpQQqQSJnJmm5p8kAXEr+E17QnDNbQ7YpszC1yHmy8ntEIl3A784f8yXufRNOYJFir+O43BaD0qfe\/E8ybQFEb\/wMzNxH0PbiaGM6fZuRxuetCSAU5wDWUE+emEiVkHNdRsVQGRAbJoutoRZnkFzwA6CyosjbLdzxuScaUYQtz\/x4oANzVRMAMzmVJ4c5nalbJW8JxLGB5MZQ9JCVYtUqHLLUdCfyU4E4HlGdK5rNarSj3ruUr+\/5kCGel2xiNIDS+c9xxjT8sS4zj8gfHVq5EP7LPuFyWrTkRmqr600UXyM+yqOFXwyU43fpvj4RXm\/bDgFfkcz2MeJFCky7zPaaOAskDznNnLRmqzyBHEcnqVNwNVWmZnSPzmAPX1eSxSk78DEv\/4pC1Zw33pmGNtPqwzbm4adGRSJMpXA1ESn83MO5nw2tlad\/f6XtHIDIIFcAd2ybubKHggF1GlVj0fZ3rkpkpXpbeP4HjVWCmpZlmt5hrqOnYKCXIoA9d5Q9eU9x0bDgEw8UsAs8Z2cGt7PGrb+Qv7bmsIIrtbYJoehXXLytxGqTyGFHgdtZ1iR39hZ3t83j6Mygm3lc680av6XxYpuCod\/9ENBc+yDd51\/4a1SVvyKfKpS1J1NPGkdCHXxqze5lGusMv4rpLextd++aXgXm4pp8tC9u7v0Y3ESoZOdsgdZjwRtBAwxPUuMR+bTiGlzmFAWnBxEgtA8qwqoeJ8fN2BBhxRSoyiJIjvIbrD\/ViWh8M6a5vCi9FaH2BHmTSkUujKoS4Ui05Uf0s+HwGa2T\/ncn+QF0sBjLTpC3akoGTkw2dqmGtGGg9JL9sxQrC3Z8P2+K0kklga\/87NYKb1gwl8HI5zrx04BnBtRZsYBSRVsc1GywvAc13NndpSo5neCnmnBd\/1I9+HIxUef4wi7p4C66Y0I2booJeN+ZoBGc\/1Y4vtaXbEsPJKJMDqB+BLCw0nSvSbDYYxB91phOhOel5GanFtMg+9nyM\/3XGQvKxO8noAo3CMoOyP7NgQIfjHvFH8Bz6xZMI7QqDGNnOF1uX5CACJ7YsOw8FPJLyQlYtZFFGiMTTrapto3gMpziUDCXvss50gfevS3poRlxl+s6OS85vpXalhuTHFjf8vGxSXfWFquDf1RFg9CUy8zk9PSl1vxgrx0OTqElj9oGT3+Vx3qZgn2bqf+592wbJFWx25hJrBNvBVEbn+OJNrZuuEh1HCoz98Rw4ULrJKM3qfOdDRZ2usK\/f4PyleqeEhwP7aUVZX0wKYFXL2UxfGiK7yY36SpPBq3Ln32t6dvMpaObtqNj+Kfr4ImRxmqQhe0B5zTHV67SrOYPC5E+e3BuEgNN6g9Xu7lBtLjFEUVfT\/s+OSTv0ASorfZmSXHEGDDlch1PtzQNW9Rg1xFAIoMDwxtBj3jiKEIJKWJ2FNgC2FjB+FshqIdc1deJTLE2ymgSABs\/nFAcJERH5Eh8SDc80l1fUqtgee0KKG7+UiEYG9HBLhxrjLYpW6nqwKOnP5iS5J75eSdcaJPQ2RCDoI48f54M\/u0C5mjF4KxZWfbF6W+LA7ItzNMe\/dXWOBsTFS8qH5T20g\/3IZenJtIlcn5ix8kqRSNhmkt78WK6PYEC8Frnz87GbQ2+TF1AIO24YEByT38EkpPfVZBJEKa7vsROTk\/wrD31hqsKtZVqrDC7NcjVOE7GiftEXF+1sA8Yo1W\/gcl71x2tP6c6oxG0OS7vSR61oZ9c4wtxmZsalZYl9wvy0wjtzOgCqQPbk69W7bNvn1ZXADwPJ8YWuzH9z1aPWM2csOqghu72ChTMW2zQtB\/qGY49wPVNjYcmbEB+443LWlsFCjcunDLVmzxVAIJIet9kbYse0PhUurR66Ele1UdzzsBsHU08\/5dPnbKk+8hDJCPyIztDNktODA9+bPmDu8JJ2UixUjK4TEzkxYFIQMx0hR4gryqlUJRl1sbbMr7VctjZdbpqLiiFuSagY+pSdIQ8GPFcdtrfWsXnDYoiBXJ\/5j+UKyYU4B2pUY38w+mhHW38VyltT030eEtueb0ipynzmIgzRdJZ\/W7TPMibiy2oykdpbb6SZ1ujx16jzA3iU7pPElUkIOkKOSxtREPgbzIlknPYKGoBQHdq0GpxSL0i9d7GU7NtI2fcQYpwP4X\/sj3JNdosmuOXAeEPYsSMWQmH+qrj6FSm9gE+WhZfWc2hGNRD7Y6OGdYaU0Q60pRVRul0FACZqyMrb5y97MpVuuqRxKzn2r7P+Z+KtgKO7S7rNMVQmOq0tVktiH\/Ws836Z6\/328nnzLauw2NXRu0qwbtytvVv0f2sBuTbqbURJET4ciDSSyF7wux7TlhQsY\/qPPlXKBUkVGHetfK0nSty5hsQc12nShr9kuLAog="}
 00178{"basic_event_id":8,"basic_event_name":"nDPI IPv4\/L4 payload detection failed","thread_id":0,"packet_id":2513,"source":"anydesk-2.pcap","alias":"nDPId-test","l4_data_len":2146}
 00497{"flow_event_id":2,"flow_event_name":"end","thread_id":0,"packet_id":2521,"source":"anydesk-2.pcap","alias":"nDPId-test","flow_id":4,"flow_packet_id":15,"flow_first_seen":1613977595407,"flow_last_seen":1613977595964,"flow_tot_l4_data_len":3652,"flow_min_l4_data_len":20,"flow_max_l4_data_len":1306,"flow_avg_l4_data_len":243,"midstream":0,"l3_proto":"ip4","src_ip":"192.168.1.178","dst_ip":"192.168.1.187","src_port":52039,"dst_port":7070,"l4_proto":"tcp","flow_datalink":1,"flow_max_packets":15}
+00499{"flow_event_id":3,"flow_event_name":"idle","thread_id":0,"packet_id":2521,"source":"anydesk-2.pcap","alias":"nDPId-test","flow_id":3,"flow_packet_id":255,"flow_first_seen":1613977595379,"flow_last_seen":1613977618224,"flow_tot_l4_data_len":21934,"flow_min_l4_data_len":20,"flow_max_l4_data_len":1480,"flow_avg_l4_data_len":86,"midstream":0,"l3_proto":"ip4","src_ip":"192.168.1.187","dst_ip":"192.168.1.178","src_port":54164,"dst_port":7070,"l4_proto":"tcp","flow_datalink":1,"flow_max_packets":15}
 00489{"flow_event_id":3,"flow_event_name":"idle","thread_id":0,"packet_id":2521,"source":"anydesk-2.pcap","alias":"nDPId-test","flow_id":2,"flow_packet_id":2,"flow_first_seen":1613977585542,"flow_last_seen":1613977585553,"flow_tot_l4_data_len":128,"flow_min_l4_data_len":56,"flow_max_l4_data_len":72,"flow_avg_l4_data_len":64,"midstream":0,"l3_proto":"ip4","src_ip":"192.168.1.187","dst_ip":"192.168.1.1","src_port":55376,"dst_port":53,"l4_proto":"udp","flow_datalink":1,"flow_max_packets":15}
 00489{"flow_event_id":3,"flow_event_name":"idle","thread_id":0,"packet_id":2521,"source":"anydesk-2.pcap","alias":"nDPId-test","flow_id":1,"flow_packet_id":2,"flow_first_seen":1613977585247,"flow_last_seen":1613977585260,"flow_tot_l4_data_len":128,"flow_min_l4_data_len":56,"flow_max_l4_data_len":72,"flow_avg_l4_data_len":64,"midstream":0,"l3_proto":"ip4","src_ip":"192.168.1.187","dst_ip":"192.168.1.1","src_port":59511,"dst_port":53,"l4_proto":"udp","flow_datalink":1,"flow_max_packets":15}
 00131{"daemon_event_id":3,"daemon_event_name":"shutdown","thread_id":0,"packet_id":2521,"source":"anydesk-2.pcap","alias":"nDPId-test"}
diff --git a/test/results/bad-dns-traffic.pcap.out b/test/results/bad-dns-traffic.pcap.out
index 6fb011978..c2e546202 100644
--- a/test/results/bad-dns-traffic.pcap.out
+++ b/test/results/bad-dns-traffic.pcap.out
@@ -43,7 +43,7 @@
 00519{"flow_id":2,"flow_packet_id":13,"packet_event_id":2,"packet_event_name":"packet-flow","thread_id":0,"packet_id":32,"source":"bad-dns-traffic.pcap","alias":"nDPId-test","pkt_oversize":false,"pkt_ts_sec":1486012642,"pkt_ts_usec":281373,"pkt_caplen":128,"pkt_type":2048,"pkt_l3_offset":14,"pkt_l4_offset":34,"pkt_len":128,"pkt_l4_len":94,"pkt":"5LMYS\/DDAhoR+f4qCABFAAByAABAADMRVXIEAgIEwKgrWwA13CIAXlbsi0KBgAABAAEAAAAAEjUwNzQwMWZkZjUyNTMyNDE3ZAxza3VsbHNlY2xhYnMDb3JnAAAFAAHADAAFAAEAAAA8ABUSYWM2YjAxZmRmNTQxN2QyNTMywB8="}
 00473{"flow_id":2,"flow_packet_id":14,"packet_event_id":2,"packet_event_name":"packet-flow","thread_id":0,"packet_id":33,"source":"bad-dns-traffic.pcap","alias":"nDPId-test","pkt_oversize":false,"pkt_ts_sec":1486012643,"pkt_ts_usec":238555,"pkt_caplen":95,"pkt_type":2048,"pkt_l3_offset":14,"pkt_l4_offset":34,"pkt_len":95,"pkt_l4_len":61,"pkt":"AhoR+f4q5LMYS\/DDCABFAABRAC9AAEARSGTAqCtbBAICBNwiADUAPaQHCm0BAAABAAAAAAAAEjc2MmIwMWZkZjUyNTMyNDE3ZAxza3VsbHNlY2xhYnMDb3JnAAAPAAE="}
 00523{"flow_id":2,"flow_packet_id":15,"packet_event_id":2,"packet_event_name":"packet-flow","thread_id":0,"packet_id":34,"source":"bad-dns-traffic.pcap","alias":"nDPId-test","pkt_oversize":false,"pkt_ts_sec":1486012643,"pkt_ts_usec":293987,"pkt_caplen":130,"pkt_type":2048,"pkt_l3_offset":14,"pkt_l4_offset":34,"pkt_len":130,"pkt_l4_len":96,"pkt":"5LMYS\/DDAhoR+f4qCABFAAB0AABAADMRVXAEAgIEwKgrWwA13CIAYLAaCm2BgAABAAEAAAAAEjc2MmIwMWZkZjUyNTMyNDE3ZAxza3VsbHNlY2xhYnMDb3JnAAAPAAHADAAPAAEAAAA8ABcAChIyOTkyMDFmZGY1NDE3ZDI1MzLAHw=="}
-00766{"flow_event_id":5,"flow_event_name":"detected","thread_id":0,"packet_id":274,"source":"bad-dns-traffic.pcap","alias":"nDPId-test","flow_id":2,"flow_packet_id":255,"flow_first_seen":1486012635073,"flow_last_seen":1486012691087,"flow_tot_l4_data_len":63345,"flow_min_l4_data_len":61,"flow_max_l4_data_len":291,"flow_avg_l4_data_len":248,"midstream":0,"l3_proto":"ip4","src_ip":"192.168.43.91","dst_ip":"4.2.2.4","src_port":56354,"dst_port":53,"l4_proto":"udp","ndpi": {"flow_risk": {"16":"Suspicious DGA domain name"},"proto":"DNS","breed":"Acceptable","category":"Network"},"dns": {"query":"c75900fdf525320021636f6d6d616e64202873697276696d65732900.skullseclabs.org","num_queries":1,"num_answers":1,"reply_code":0,"query_type":16,"rsp_type":16,"rsp_addr":"0.0.0.0"}}
+00774{"flow_event_id":6,"flow_event_name":"detection-update","thread_id":0,"packet_id":274,"source":"bad-dns-traffic.pcap","alias":"nDPId-test","flow_id":2,"flow_packet_id":255,"flow_first_seen":1486012635073,"flow_last_seen":1486012691087,"flow_tot_l4_data_len":63345,"flow_min_l4_data_len":61,"flow_max_l4_data_len":291,"flow_avg_l4_data_len":248,"midstream":0,"l3_proto":"ip4","src_ip":"192.168.43.91","dst_ip":"4.2.2.4","src_port":56354,"dst_port":53,"l4_proto":"udp","ndpi": {"flow_risk": {"16":"Suspicious DGA domain name"},"proto":"DNS","breed":"Acceptable","category":"Network"},"dns": {"query":"c75900fdf525320021636f6d6d616e64202873697276696d65732900.skullseclabs.org","num_queries":1,"num_answers":1,"reply_code":0,"query_type":16,"rsp_type":16,"rsp_addr":"0.0.0.0"}}
 00476{"flow_event_id":1,"flow_event_name":"new","thread_id":0,"packet_id":369,"source":"bad-dns-traffic.pcap","alias":"nDPId-test","flow_id":3,"flow_packet_id":1,"flow_first_seen":1486012730177,"flow_last_seen":0,"flow_tot_l4_data_len":99,"flow_min_l4_data_len":99,"flow_max_l4_data_len":99,"flow_avg_l4_data_len":99,"midstream":0,"l3_proto":"ip4","src_ip":"192.168.43.91","dst_ip":"4.2.2.4","src_port":46961,"dst_port":53,"l4_proto":"udp","flow_datalink":1,"flow_max_packets":15}
 00529{"flow_id":3,"flow_packet_id":1,"packet_event_id":2,"packet_event_name":"packet-flow","thread_id":0,"packet_id":369,"source":"bad-dns-traffic.pcap","alias":"nDPId-test","pkt_oversize":false,"pkt_ts_sec":1486012730,"pkt_ts_usec":177697,"pkt_caplen":133,"pkt_type":2048,"pkt_l3_offset":14,"pkt_l4_offset":34,"pkt_len":133,"pkt_l4_len":99,"pkt":"AhoR+f4q5LMYS\/DDCABFAAB3Lk5AAEARGh\/AqCtbBAICBLdxADUAYz49\/HsBAAABAAAAAAAAOGEwNTcwMGU2ZGE4MzUxMDAwMTYzNmY2ZTczNmY2YzY1MjAyODczNjk3Mjc2Njk2ZDY1NzMyOTAwDHNrdWxsc2VjbGFicwNvcmcAAA8AAQ=="}
 00746{"flow_event_id":5,"flow_event_name":"detected","thread_id":0,"packet_id":369,"source":"bad-dns-traffic.pcap","alias":"nDPId-test","flow_id":3,"flow_packet_id":1,"flow_first_seen":1486012730177,"flow_last_seen":0,"flow_tot_l4_data_len":99,"flow_min_l4_data_len":99,"flow_max_l4_data_len":99,"flow_avg_l4_data_len":99,"midstream":0,"l3_proto":"ip4","src_ip":"192.168.43.91","dst_ip":"4.2.2.4","src_port":46961,"dst_port":53,"l4_proto":"udp","ndpi": {"flow_risk": {"16":"Suspicious DGA domain name"},"proto":"DNS","breed":"Acceptable","category":"Network"},"dns": {"query":"a05700e6da83510001636f6e736f6c65202873697276696d65732900.skullseclabs.org","num_queries":0,"num_answers":0,"reply_code":0,"query_type":15,"rsp_type":0,"rsp_addr":"0.0.0.0"}}
@@ -66,5 +66,6 @@
 00474{"flow_id":3,"flow_packet_id":13,"packet_event_id":2,"packet_event_name":"packet-flow","thread_id":0,"packet_id":381,"source":"bad-dns-traffic.pcap","alias":"nDPId-test","pkt_oversize":false,"pkt_ts_sec":1486012733,"pkt_ts_usec":574897,"pkt_caplen":95,"pkt_type":2048,"pkt_l3_offset":14,"pkt_l4_offset":34,"pkt_len":95,"pkt_l4_len":61,"pkt":"AhoR+f4q5LMYS\/DDCABFAABRMElAAEARGErAqCtbBAICBLdxADUAPeYHvL4BAAABAAAAAAAAEjU0NWIwMWU2ZGE4M2JmNmVhMgxza3VsbHNlY2xhYnMDb3JnAAAPAAE="}
 00524{"flow_id":3,"flow_packet_id":14,"packet_event_id":2,"packet_event_name":"packet-flow","thread_id":0,"packet_id":382,"source":"bad-dns-traffic.pcap","alias":"nDPId-test","pkt_oversize":false,"pkt_ts_sec":1486012733,"pkt_ts_usec":669835,"pkt_caplen":130,"pkt_type":2048,"pkt_l3_offset":14,"pkt_l4_offset":34,"pkt_len":130,"pkt_l4_len":96,"pkt":"5LMYS\/DDAhoR+f4qCABFAAB0AABAADMRVXAEAgIEwKgrWwA1t3EAYDm3vL6BgAABAAEAAAAAEjU0NWIwMWU2ZGE4M2JmNmVhMgxza3VsbHNlY2xhYnMDb3JnAAAPAAHADAAPAAEAAAA8ABcAChJhOGRkMDFlNmRhNmVhMjgzYmbAHw=="}
 00494{"flow_event_id":3,"flow_event_name":"idle","thread_id":0,"packet_id":382,"source":"bad-dns-traffic.pcap","alias":"nDPId-test","flow_id":3,"flow_packet_id":14,"flow_first_seen":1486012730177,"flow_last_seen":1486012733669,"flow_tot_l4_data_len":1607,"flow_min_l4_data_len":61,"flow_max_l4_data_len":289,"flow_avg_l4_data_len":114,"midstream":0,"l3_proto":"ip4","src_ip":"192.168.43.91","dst_ip":"4.2.2.4","src_port":46961,"dst_port":53,"l4_proto":"udp","flow_datalink":1,"flow_max_packets":15}
+00496{"flow_event_id":3,"flow_event_name":"idle","thread_id":0,"packet_id":382,"source":"bad-dns-traffic.pcap","alias":"nDPId-test","flow_id":2,"flow_packet_id":255,"flow_first_seen":1486012635073,"flow_last_seen":1486012727540,"flow_tot_l4_data_len":63345,"flow_min_l4_data_len":61,"flow_max_l4_data_len":291,"flow_avg_l4_data_len":248,"midstream":0,"l3_proto":"ip4","src_ip":"192.168.43.91","dst_ip":"4.2.2.4","src_port":56354,"dst_port":53,"l4_proto":"udp","flow_datalink":1,"flow_max_packets":15}
 00493{"flow_event_id":3,"flow_event_name":"idle","thread_id":0,"packet_id":382,"source":"bad-dns-traffic.pcap","alias":"nDPId-test","flow_id":1,"flow_packet_id":19,"flow_first_seen":1486012623234,"flow_last_seen":1486012630741,"flow_tot_l4_data_len":1772,"flow_min_l4_data_len":61,"flow_max_l4_data_len":195,"flow_avg_l4_data_len":93,"midstream":0,"l3_proto":"ip4","src_ip":"192.168.43.91","dst_ip":"4.2.2.4","src_port":35966,"dst_port":53,"l4_proto":"udp","flow_datalink":1,"flow_max_packets":15}
 00136{"daemon_event_id":3,"daemon_event_name":"shutdown","thread_id":0,"packet_id":382,"source":"bad-dns-traffic.pcap","alias":"nDPId-test"}
diff --git a/test/results/bittorrent_ip.pcap.out b/test/results/bittorrent_ip.pcap.out
index f5a3e0752..efcd11c47 100644
--- a/test/results/bittorrent_ip.pcap.out
+++ b/test/results/bittorrent_ip.pcap.out
@@ -35,4 +35,5 @@
 00590{"flow_event_id":5,"flow_event_name":"detected","thread_id":0,"packet_id":303,"source":"bittorrent_ip.pcap","alias":"nDPId-test","flow_id":2,"flow_packet_id":255,"flow_first_seen":1492508991649,"flow_last_seen":1492508992859,"flow_tot_l4_data_len":267352,"flow_min_l4_data_len":32,"flow_max_l4_data_len":1480,"flow_avg_l4_data_len":1048,"midstream":1,"l3_proto":"ip4","src_ip":"77.222.174.20","dst_ip":"10.0.0.14","src_port":2866,"dst_port":46610,"l4_proto":"tcp","ndpi": {"proto":"BitTorrent","breed":"Acceptable","category":"Download-FileTransfer-FileSharing"},"bittorrent": {"hash":""}}
 00586{"flow_event_id":4,"flow_event_name":"guessed","thread_id":0,"packet_id":479,"source":"bittorrent_ip.pcap","alias":"nDPId-test","flow_id":1,"flow_packet_id":48,"flow_first_seen":1492508985380,"flow_last_seen":1492508985594,"flow_tot_l4_data_len":36300,"flow_min_l4_data_len":32,"flow_max_l4_data_len":1480,"flow_avg_l4_data_len":756,"midstream":1,"l3_proto":"ip4","src_ip":"185.56.20.36","dst_ip":"10.0.0.14","src_port":53646,"dst_port":35030,"l4_proto":"tcp","ndpi": {"proto":"BitTorrent","breed":"Acceptable","category":"Download-FileTransfer-FileSharing"},"bittorrent": {"hash":""}}
 00498{"flow_event_id":3,"flow_event_name":"idle","thread_id":0,"packet_id":479,"source":"bittorrent_ip.pcap","alias":"nDPId-test","flow_id":1,"flow_packet_id":48,"flow_first_seen":1492508985380,"flow_last_seen":1492508985594,"flow_tot_l4_data_len":36300,"flow_min_l4_data_len":32,"flow_max_l4_data_len":1480,"flow_avg_l4_data_len":756,"midstream":1,"l3_proto":"ip4","src_ip":"185.56.20.36","dst_ip":"10.0.0.14","src_port":53646,"dst_port":35030,"l4_proto":"tcp","flow_datalink":1,"flow_max_packets":15}
+00501{"flow_event_id":3,"flow_event_name":"idle","thread_id":0,"packet_id":479,"source":"bittorrent_ip.pcap","alias":"nDPId-test","flow_id":2,"flow_packet_id":255,"flow_first_seen":1492508991649,"flow_last_seen":1492508994096,"flow_tot_l4_data_len":267352,"flow_min_l4_data_len":32,"flow_max_l4_data_len":1480,"flow_avg_l4_data_len":1048,"midstream":1,"l3_proto":"ip4","src_ip":"77.222.174.20","dst_ip":"10.0.0.14","src_port":2866,"dst_port":46610,"l4_proto":"tcp","flow_datalink":1,"flow_max_packets":15}
 00134{"daemon_event_id":3,"daemon_event_name":"shutdown","thread_id":0,"packet_id":479,"source":"bittorrent_ip.pcap","alias":"nDPId-test"}
diff --git a/test/results/dns-tunnel-iodine.pcap.out b/test/results/dns-tunnel-iodine.pcap.out
index bab9ceb7c..d00ff8565 100644
--- a/test/results/dns-tunnel-iodine.pcap.out
+++ b/test/results/dns-tunnel-iodine.pcap.out
@@ -21,5 +21,6 @@
 00508{"flow_id":1,"flow_packet_id":13,"packet_event_id":2,"packet_event_name":"packet-flow","thread_id":0,"packet_id":13,"source":"dns-tunnel-iodine.pcap","alias":"nDPId-test","pkt_oversize":false,"pkt_ts_sec":1282356640,"pkt_ts_usec":58865,"pkt_caplen":118,"pkt_type":2048,"pkt_l3_offset":14,"pkt_l4_offset":34,"pkt_len":118,"pkt_l4_len":84,"pkt":"CAAnx266CAAnnOC0CABFAABoAABAAEARIlQKAAIeCgACFK5fADUAVBazx8oBAAABAAAAAAABJHppMWFhQTAxMjM0NTY3ODm8vb6\/wMHCw8TFxsfIycrLzM3OzwZwaXJhdGUDc2VhAAAKAAEAACkQAAAAgAAAAA=="}
 00557{"flow_id":1,"flow_packet_id":14,"packet_event_id":2,"packet_event_name":"packet-flow","thread_id":0,"packet_id":14,"source":"dns-tunnel-iodine.pcap","alias":"nDPId-test","pkt_oversize":false,"pkt_ts_sec":1282356640,"pkt_ts_usec":58974,"pkt_caplen":156,"pkt_type":2048,"pkt_l3_offset":14,"pkt_l4_offset":34,"pkt_len":156,"pkt_l4_len":122,"pkt":"CAAnnOC0CAAnx266CABFAACOAABAAEARIi4KAAIUCgACHgA1rl8AegzWx8qEAAABAAEAAAAAJHppMWFhQTAxMjM0NTY3ODm8vb6\/wMHCw8TFxsfIycrLzM3OzwZwaXJhdGUDc2VhAAAKAAHADAAKAAEAAAAAACV6aTFhYUEwMTIzNDU2Nzg5vL2+v8DBwsPExcbHyMnKy8zNzs8u"}
 00531{"flow_id":1,"flow_packet_id":15,"packet_event_id":2,"packet_event_name":"packet-flow","thread_id":0,"packet_id":15,"source":"dns-tunnel-iodine.pcap","alias":"nDPId-test","pkt_oversize":false,"pkt_ts_sec":1282356640,"pkt_ts_usec":59078,"pkt_caplen":134,"pkt_type":2048,"pkt_l3_offset":14,"pkt_l4_offset":34,"pkt_len":134,"pkt_l4_len":100,"pkt":"CAAnx266CAAnnOC0CABFAAB4AABAAEARIkQKAAIeCgACFK5fADUAZN9j5fkBAAABAAAAAAABNHppMWJhQdDR0tPU1dbX2Nna29zd3t\/g4eLj5OXm5+jp6uvs7e7v8PHy8\/T19vf4+fr7\/P0GcGlyYXRlA3NlYQAACgABAAApEAAAAIAAAAA="}
-00741{"flow_event_id":5,"flow_event_name":"detected","thread_id":0,"packet_id":259,"source":"dns-tunnel-iodine.pcap","alias":"nDPId-test","flow_id":1,"flow_packet_id":255,"flow_first_seen":1282356640051,"flow_last_seen":1282356654812,"flow_tot_l4_data_len":37534,"flow_min_l4_data_len":48,"flow_max_l4_data_len":1478,"flow_avg_l4_data_len":147,"midstream":0,"l3_proto":"ip4","src_ip":"10.0.2.30","dst_ip":"10.0.2.20","src_port":44639,"dst_port":53,"l4_proto":"udp","ndpi": {"flow_risk": {"16":"Suspicious DGA domain name","23":"Suspicious DNS traffic"},"proto":"DNS","breed":"Acceptable","category":"Network"},"dns": {"query":"yrbi02.pirate.sea","num_queries":1,"num_answers":1,"reply_code":0,"query_type":10,"rsp_type":10,"rsp_addr":"0.0.0.0"}}
+00749{"flow_event_id":6,"flow_event_name":"detection-update","thread_id":0,"packet_id":259,"source":"dns-tunnel-iodine.pcap","alias":"nDPId-test","flow_id":1,"flow_packet_id":255,"flow_first_seen":1282356640051,"flow_last_seen":1282356654812,"flow_tot_l4_data_len":37534,"flow_min_l4_data_len":48,"flow_max_l4_data_len":1478,"flow_avg_l4_data_len":147,"midstream":0,"l3_proto":"ip4","src_ip":"10.0.2.30","dst_ip":"10.0.2.20","src_port":44639,"dst_port":53,"l4_proto":"udp","ndpi": {"flow_risk": {"16":"Suspicious DGA domain name","23":"Suspicious DNS traffic"},"proto":"DNS","breed":"Acceptable","category":"Network"},"dns": {"query":"yrbi02.pirate.sea","num_queries":1,"num_answers":1,"reply_code":0,"query_type":10,"rsp_type":10,"rsp_addr":"0.0.0.0"}}
+00497{"flow_event_id":3,"flow_event_name":"idle","thread_id":0,"packet_id":438,"source":"dns-tunnel-iodine.pcap","alias":"nDPId-test","flow_id":1,"flow_packet_id":255,"flow_first_seen":1282356640051,"flow_last_seen":1282356664538,"flow_tot_l4_data_len":37534,"flow_min_l4_data_len":48,"flow_max_l4_data_len":1478,"flow_avg_l4_data_len":147,"midstream":0,"l3_proto":"ip4","src_ip":"10.0.2.30","dst_ip":"10.0.2.20","src_port":44639,"dst_port":53,"l4_proto":"udp","flow_datalink":1,"flow_max_packets":15}
 00138{"daemon_event_id":3,"daemon_event_name":"shutdown","thread_id":0,"packet_id":438,"source":"dns-tunnel-iodine.pcap","alias":"nDPId-test"}
diff --git a/test/results/dns_exfiltration.pcap.out b/test/results/dns_exfiltration.pcap.out
index 2f085b840..f0c521936 100644
--- a/test/results/dns_exfiltration.pcap.out
+++ b/test/results/dns_exfiltration.pcap.out
@@ -21,5 +21,6 @@
 00483{"flow_id":1,"flow_packet_id":13,"packet_event_id":2,"packet_event_name":"packet-flow","thread_id":0,"packet_id":13,"source":"dns_exfiltration.pcap","alias":"nDPId-test","pkt_oversize":false,"pkt_ts_sec":1580978151,"pkt_ts_usec":800983,"pkt_caplen":101,"pkt_type":2048,"pkt_l3_offset":14,"pkt_l4_offset":34,"pkt_len":101,"pkt_l4_len":67,"pkt":"qqru7hERjNzURr7ECABFAABXeuJAAD8RAADAqNw4wKjLp9w1ADUAQymGXxkBAAABAAAAAAAABmRuc2NhdCJmYjhiMDFmNTAwMmZjMDE3ZTYxYmRhMDAwNWQ3YTZhZWFjAAAQAAE="}
 00548{"flow_id":1,"flow_packet_id":14,"packet_event_id":2,"packet_event_name":"packet-flow","thread_id":0,"packet_id":14,"source":"dns_exfiltration.pcap","alias":"nDPId-test","pkt_oversize":false,"pkt_ts_sec":1580978151,"pkt_ts_usec":802508,"pkt_caplen":148,"pkt_type":2048,"pkt_l3_offset":14,"pkt_l4_offset":34,"pkt_len":148,"pkt_l4_len":114,"pkt":"jNzURr7Eqqru7hERCABFAACGPjJAAD8R1APAqMunwKjcOAA13DUAckeuXxmBgAABAAEAAAAABmRuc2NhdCJmYjhiMDFmNTAwMmZjMDE3ZTYxYmRhMDAwNWQ3YTZhZWFjAAAQAAHADAAQAAEAAAA8ACMiYTYzZjAxZjUwMDc0MjhjMzBlMWMwYWZmZmZmZmQzYWE0Yg=="}
 00483{"flow_id":1,"flow_packet_id":15,"packet_event_id":2,"packet_event_name":"packet-flow","thread_id":0,"packet_id":15,"source":"dns_exfiltration.pcap","alias":"nDPId-test","pkt_oversize":false,"pkt_ts_sec":1580978152,"pkt_ts_usec":810482,"pkt_caplen":101,"pkt_type":2048,"pkt_l3_offset":14,"pkt_l4_offset":34,"pkt_len":101,"pkt_l4_len":67,"pkt":"qqru7hERjNzURr7ECABFAABXezVAAD8RAADAqNw4wKjLp9w1ADUAQymG420BAAABAAAAAAAABmRuc2NhdCJjNGY5MDFmNTAwNDcxY2Q2ODNlZWQwMDAwNmY5MDdmMGY0AAAPAAE="}
-00779{"flow_event_id":5,"flow_event_name":"detected","thread_id":0,"packet_id":255,"source":"dns_exfiltration.pcap","alias":"nDPId-test","flow_id":1,"flow_packet_id":255,"flow_first_seen":1580978146717,"flow_last_seen":1580978206666,"flow_tot_l4_data_len":50136,"flow_min_l4_data_len":67,"flow_max_l4_data_len":352,"flow_avg_l4_data_len":196,"midstream":0,"l3_proto":"ip4","src_ip":"192.168.220.56","dst_ip":"192.168.203.167","src_port":56373,"dst_port":53,"l4_proto":"udp","ndpi": {"flow_risk": {"16":"Suspicious DGA domain name"},"proto":"DNS","breed":"Acceptable","category":"Network"},"dns": {"query":"dnscat.a35c00f5005703c8b1b8cd000118b52347aeb1d73340c97cca43c34b27cf.edf0dbda","num_queries":1,"num_answers":1,"reply_code":0,"query_type":15,"rsp_type":15,"rsp_addr":"0.0.0.0"}}
+00787{"flow_event_id":6,"flow_event_name":"detection-update","thread_id":0,"packet_id":255,"source":"dns_exfiltration.pcap","alias":"nDPId-test","flow_id":1,"flow_packet_id":255,"flow_first_seen":1580978146717,"flow_last_seen":1580978206666,"flow_tot_l4_data_len":50136,"flow_min_l4_data_len":67,"flow_max_l4_data_len":352,"flow_avg_l4_data_len":196,"midstream":0,"l3_proto":"ip4","src_ip":"192.168.220.56","dst_ip":"192.168.203.167","src_port":56373,"dst_port":53,"l4_proto":"udp","ndpi": {"flow_risk": {"16":"Suspicious DGA domain name"},"proto":"DNS","breed":"Acceptable","category":"Network"},"dns": {"query":"dnscat.a35c00f5005703c8b1b8cd000118b52347aeb1d73340c97cca43c34b27cf.edf0dbda","num_queries":1,"num_answers":1,"reply_code":0,"query_type":15,"rsp_type":15,"rsp_addr":"0.0.0.0"}}
+00506{"flow_event_id":3,"flow_event_name":"idle","thread_id":0,"packet_id":300,"source":"dns_exfiltration.pcap","alias":"nDPId-test","flow_id":1,"flow_packet_id":255,"flow_first_seen":1580978146717,"flow_last_seen":1580978206707,"flow_tot_l4_data_len":50136,"flow_min_l4_data_len":67,"flow_max_l4_data_len":352,"flow_avg_l4_data_len":196,"midstream":0,"l3_proto":"ip4","src_ip":"192.168.220.56","dst_ip":"192.168.203.167","src_port":56373,"dst_port":53,"l4_proto":"udp","flow_datalink":1,"flow_max_packets":15}
 00137{"daemon_event_id":3,"daemon_event_name":"shutdown","thread_id":0,"packet_id":300,"source":"dns_exfiltration.pcap","alias":"nDPId-test"}
diff --git a/test/results/ftp.pcap.out b/test/results/ftp.pcap.out
index 324be94bb..57c31eebe 100644
--- a/test/results/ftp.pcap.out
+++ b/test/results/ftp.pcap.out
@@ -44,6 +44,7 @@
 00420{"flow_id":3,"flow_packet_id":14,"packet_event_id":2,"packet_event_name":"packet-flow","thread_id":0,"packet_id":82,"source":"ftp.pcap","alias":"nDPId-test","pkt_oversize":false,"pkt_ts_sec":1552590241,"pkt_ts_usec":605580,"pkt_caplen":66,"pkt_type":2048,"pkt_l3_offset":14,"pkt_l4_offset":34,"pkt_len":66,"pkt_l4_len":32,"pkt":"EBMx8Tl2xCwDBkn+CABFCAA0AABAAEAGAADAqAHUWoJGScYIX8sNBxpPDE6a8YAQEABjbgAAAQEICjtXspsSaAHT"}
 02347{"flow_id":3,"flow_packet_id":15,"packet_event_id":2,"packet_event_name":"packet-flow","thread_id":0,"packet_id":83,"source":"ftp.pcap","alias":"nDPId-test","pkt_oversize":false,"pkt_ts_sec":1552590241,"pkt_ts_usec":605595,"pkt_caplen":1506,"pkt_type":2048,"pkt_l3_offset":14,"pkt_l4_offset":34,"pkt_len":1506,"pkt_l4_len":1472,"pkt":"xCwDBkn+EBMx8Tl2CABFAAXUeOhAADYGYvRagkZJwKgB1F\/LxggMTprxDQcaT4AQAAMXWQAAAQEIChJoAdM7V7J9AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"}
 00508{"flow_event_id":7,"flow_event_name":"not-detected","thread_id":0,"packet_id":323,"source":"ftp.pcap","alias":"nDPId-test","flow_id":3,"flow_packet_id":255,"flow_first_seen":1552590241545,"flow_last_seen":1552590241726,"flow_tot_l4_data_len":224192,"flow_min_l4_data_len":32,"flow_max_l4_data_len":1472,"flow_avg_l4_data_len":879,"midstream":0,"l3_proto":"ip4","src_ip":"192.168.1.212","dst_ip":"90.130.70.73","src_port":50696,"dst_port":24523,"l4_proto":"tcp","ndpi": {"proto":"Unknown","breed":"Unrated"}}
+00494{"flow_event_id":2,"flow_event_name":"end","thread_id":0,"packet_id":1192,"source":"ftp.pcap","alias":"nDPId-test","flow_id":3,"flow_packet_id":255,"flow_first_seen":1552590241545,"flow_last_seen":1552590241878,"flow_tot_l4_data_len":224192,"flow_min_l4_data_len":32,"flow_max_l4_data_len":1472,"flow_avg_l4_data_len":879,"midstream":0,"l3_proto":"ip4","src_ip":"192.168.1.212","dst_ip":"90.130.70.73","src_port":50696,"dst_port":24523,"l4_proto":"tcp","flow_datalink":1,"flow_max_packets":15}
 00486{"flow_event_id":2,"flow_event_name":"end","thread_id":0,"packet_id":1192,"source":"ftp.pcap","alias":"nDPId-test","flow_id":1,"flow_packet_id":68,"flow_first_seen":1552590234892,"flow_last_seen":1552590243371,"flow_tot_l4_data_len":3259,"flow_min_l4_data_len":32,"flow_max_l4_data_len":273,"flow_avg_l4_data_len":47,"midstream":0,"l3_proto":"ip4","src_ip":"192.168.1.212","dst_ip":"90.130.70.73","src_port":50694,"dst_port":21,"l4_proto":"tcp","flow_datalink":1,"flow_max_packets":15}
 00490{"flow_event_id":2,"flow_event_name":"end","thread_id":0,"packet_id":1192,"source":"ftp.pcap","alias":"nDPId-test","flow_id":2,"flow_packet_id":9,"flow_first_seen":1552590236580,"flow_last_seen":1552590236666,"flow_tot_l4_data_len":1513,"flow_min_l4_data_len":32,"flow_max_l4_data_len":1237,"flow_avg_l4_data_len":168,"midstream":0,"l3_proto":"ip4","src_ip":"192.168.1.212","dst_ip":"90.130.70.73","src_port":50695,"dst_port":25685,"l4_proto":"tcp","flow_datalink":1,"flow_max_packets":15}
 00125{"daemon_event_id":3,"daemon_event_name":"shutdown","thread_id":0,"packet_id":1192,"source":"ftp.pcap","alias":"nDPId-test"}
diff --git a/test/results/memcached.cap.out b/test/results/memcached.cap.out
index de834d3d4..a528453d6 100644
--- a/test/results/memcached.cap.out
+++ b/test/results/memcached.cap.out
@@ -1,3 +1,4 @@
+00385{"daemon_event_id":1,"daemon_event_name":"init","thread_id":0,"packet_id":0,"source":"memcached.cap","alias":"nDPId-test","max-flows-per-thread":2048,"max-idle-flows-per-thread":256,"tick-resolution":1000,"reader-thread-count":1,"idle-scan-period":10000,"max-idle-time":600000,"tcp-max-post-end-flow-time":60000,"max-packets-per-flow-to-send":15,"max-packets-per-flow-to-process":255}
 00468{"flow_event_id":1,"flow_event_name":"new","thread_id":0,"packet_id":1,"source":"memcached.cap","alias":"nDPId-test","flow_id":1,"flow_packet_id":1,"flow_first_seen":1534343745954,"flow_last_seen":0,"flow_tot_l4_data_len":40,"flow_min_l4_data_len":40,"flow_max_l4_data_len":40,"flow_avg_l4_data_len":40,"midstream":0,"l3_proto":"ip4","src_ip":"127.0.0.1","dst_ip":"127.0.0.1","src_port":59604,"dst_port":11211,"l4_proto":"tcp","flow_datalink":1,"flow_max_packets":15}
 00437{"flow_id":1,"flow_packet_id":1,"packet_event_id":2,"packet_event_name":"packet-flow","thread_id":0,"packet_id":1,"source":"memcached.cap","alias":"nDPId-test","pkt_oversize":false,"pkt_ts_sec":1534343745,"pkt_ts_usec":954071,"pkt_caplen":74,"pkt_type":2048,"pkt_l3_offset":14,"pkt_l4_offset":34,"pkt_len":74,"pkt_l4_len":40,"pkt":"AAAAAAAAAAAAAAAACABFAAA8pT5AAEAGl3t\/AAABfwAAAejUK8sskd7QAAAAAKACqqr+MAAAAgT\/1wQCCAopIHvuAAAAAAEDAwc="}
 00437{"flow_id":1,"flow_packet_id":2,"packet_event_id":2,"packet_event_name":"packet-flow","thread_id":0,"packet_id":2,"source":"memcached.cap","alias":"nDPId-test","pkt_oversize":false,"pkt_ts_sec":1534343745,"pkt_ts_usec":954090,"pkt_caplen":74,"pkt_type":2048,"pkt_l3_offset":14,"pkt_l4_offset":34,"pkt_len":74,"pkt_l4_len":40,"pkt":"AAAAAAAAAAAAAAAACABFAAA8AABAAEAGPLp\/AAABfwAAASvL6NTLJnx6LJHe0aASqqr+MAAAAgT\/1wQCCAopIHvuKSB77gEDAwc="}
@@ -10,3 +11,5 @@
 00424{"flow_id":1,"flow_packet_id":8,"packet_event_id":2,"packet_event_name":"packet-flow","thread_id":0,"packet_id":8,"source":"memcached.cap","alias":"nDPId-test","pkt_oversize":false,"pkt_ts_sec":1534343745,"pkt_ts_usec":954689,"pkt_caplen":66,"pkt_type":2048,"pkt_l3_offset":14,"pkt_l4_offset":34,"pkt_len":66,"pkt_l4_len":32,"pkt":"AAAAAAAAAAAAAAAACABFAAA0pUJAAEAGl39\/AAABfwAAAejUK8sskd7YyyaAf4ARAWb+KAAAAQEICikge+4pIHvu"}
 00425{"flow_id":1,"flow_packet_id":9,"packet_event_id":2,"packet_event_name":"packet-flow","thread_id":0,"packet_id":9,"source":"memcached.cap","alias":"nDPId-test","pkt_oversize":false,"pkt_ts_sec":1534343745,"pkt_ts_usec":954737,"pkt_caplen":66,"pkt_type":2048,"pkt_l3_offset":14,"pkt_l4_offset":34,"pkt_len":66,"pkt_l4_len":32,"pkt":"AAAAAAAAAAAAAAAACABFAAA0B5dAAEAGNSt\/AAABfwAAASvL6NTLJoB\/LJHe2YARAVb+KAAAAQEICikge+4pIHvu"}
 00426{"flow_id":1,"flow_packet_id":10,"packet_event_id":2,"packet_event_name":"packet-flow","thread_id":0,"packet_id":10,"source":"memcached.cap","alias":"nDPId-test","pkt_oversize":false,"pkt_ts_sec":1534343745,"pkt_ts_usec":954749,"pkt_caplen":66,"pkt_type":2048,"pkt_l3_offset":14,"pkt_l4_offset":34,"pkt_len":66,"pkt_l4_len":32,"pkt":"AAAAAAAAAAAAAAAACABFAAA0pUNAAEAGl35\/AAABfwAAAejUK8sskd7ZyyaAgIAQAWb+KAAAAQEICikge+4pIHvu"}
+00487{"flow_event_id":2,"flow_event_name":"end","thread_id":0,"packet_id":10,"source":"memcached.cap","alias":"nDPId-test","flow_id":1,"flow_packet_id":10,"flow_first_seen":1534343745954,"flow_last_seen":1534343745954,"flow_tot_l4_data_len":1371,"flow_min_l4_data_len":32,"flow_max_l4_data_len":1060,"flow_avg_l4_data_len":137,"midstream":0,"l3_proto":"ip4","src_ip":"127.0.0.1","dst_ip":"127.0.0.1","src_port":59604,"dst_port":11211,"l4_proto":"tcp","flow_datalink":1,"flow_max_packets":15}
+00128{"daemon_event_id":3,"daemon_event_name":"shutdown","thread_id":0,"packet_id":10,"source":"memcached.cap","alias":"nDPId-test"}
diff --git a/test/results/pps.pcap.out b/test/results/pps.pcap.out
index cd37d484a..3ef12fa8f 100644
--- a/test/results/pps.pcap.out
+++ b/test/results/pps.pcap.out
@@ -649,6 +649,7 @@
 00496{"flow_event_id":2,"flow_event_name":"end","thread_id":0,"packet_id":2557,"source":"pps.pcap","alias":"nDPId-test","flow_id":107,"flow_packet_id":542,"flow_first_seen":1467353198532,"flow_last_seen":1467353199507,"flow_tot_l4_data_len":691957,"flow_min_l4_data_len":269,"flow_max_l4_data_len":1280,"flow_avg_l4_data_len":1276,"midstream":1,"l3_proto":"ip4","src_ip":"192.168.115.8","dst_ip":"223.26.106.20","src_port":50780,"dst_port":80,"l4_proto":"tcp","flow_datalink":1,"flow_max_packets":15}
 00478{"flow_event_id":3,"flow_event_name":"idle","thread_id":0,"packet_id":2557,"source":"pps.pcap","alias":"nDPId-test","flow_id":109,"flow_packet_id":1,"flow_first_seen":1467353200271,"flow_last_seen":0,"flow_tot_l4_data_len":269,"flow_min_l4_data_len":269,"flow_max_l4_data_len":269,"flow_avg_l4_data_len":269,"midstream":1,"l3_proto":"ip4","src_ip":"192.168.115.8","dst_ip":"223.26.106.20","src_port":50781,"dst_port":80,"l4_proto":"tcp","flow_datalink":1,"flow_max_packets":15}
 00490{"flow_event_id":3,"flow_event_name":"idle","thread_id":0,"packet_id":2557,"source":"pps.pcap","alias":"nDPId-test","flow_id":88,"flow_packet_id":2,"flow_first_seen":1467353190168,"flow_last_seen":1467353190235,"flow_tot_l4_data_len":330,"flow_min_l4_data_len":165,"flow_max_l4_data_len":165,"flow_avg_l4_data_len":165,"midstream":1,"l3_proto":"ip4","src_ip":"202.108.14.219","dst_ip":"192.168.115.8","src_port":80,"dst_port":50295,"l4_proto":"tcp","flow_datalink":1,"flow_max_packets":15}
+00493{"flow_event_id":3,"flow_event_name":"idle","thread_id":0,"packet_id":2557,"source":"pps.pcap","alias":"nDPId-test","flow_id":3,"flow_packet_id":255,"flow_first_seen":1467353136439,"flow_last_seen":1467353136982,"flow_tot_l4_data_len":96799,"flow_min_l4_data_len":45,"flow_max_l4_data_len":1073,"flow_avg_l4_data_len":379,"midstream":0,"l3_proto":"ip4","src_ip":"192.168.115.8","dst_ip":"114.42.0.158","src_port":22793,"dst_port":7716,"l4_proto":"udp","flow_datalink":1,"flow_max_packets":15}
 00492{"flow_event_id":3,"flow_event_name":"idle","thread_id":0,"packet_id":2557,"source":"pps.pcap","alias":"nDPId-test","flow_id":89,"flow_packet_id":5,"flow_first_seen":1467353190178,"flow_last_seen":1467353202194,"flow_tot_l4_data_len":705,"flow_min_l4_data_len":141,"flow_max_l4_data_len":141,"flow_avg_l4_data_len":141,"midstream":0,"l3_proto":"ip4","src_ip":"192.168.5.28","dst_ip":"239.255.255.250","src_port":60023,"dst_port":1900,"l4_proto":"udp","flow_datalink":1,"flow_max_packets":15}
 00501{"flow_event_id":7,"flow_event_name":"not-detected","thread_id":0,"packet_id":2557,"source":"pps.pcap","alias":"nDPId-test","flow_id":12,"flow_packet_id":2,"flow_first_seen":1467353136833,"flow_last_seen":1467353136833,"flow_tot_l4_data_len":64,"flow_min_l4_data_len":32,"flow_max_l4_data_len":32,"flow_avg_l4_data_len":32,"midstream":0,"l3_proto":"ip4","src_ip":"192.168.115.8","dst_ip":"210.44.171.1","src_port":22793,"dst_port":29702,"l4_proto":"udp","ndpi": {"proto":"Unknown","breed":"Unrated"}}
 00487{"flow_event_id":3,"flow_event_name":"idle","thread_id":0,"packet_id":2557,"source":"pps.pcap","alias":"nDPId-test","flow_id":12,"flow_packet_id":2,"flow_first_seen":1467353136833,"flow_last_seen":1467353136833,"flow_tot_l4_data_len":64,"flow_min_l4_data_len":32,"flow_max_l4_data_len":32,"flow_avg_l4_data_len":32,"midstream":0,"l3_proto":"ip4","src_ip":"192.168.115.8","dst_ip":"210.44.171.1","src_port":22793,"dst_port":29702,"l4_proto":"udp","flow_datalink":1,"flow_max_packets":15}
diff --git a/test/results/ps_vue.pcap.out b/test/results/ps_vue.pcap.out
index 2ccdda05f..126482522 100644
--- a/test/results/ps_vue.pcap.out
+++ b/test/results/ps_vue.pcap.out
@@ -46,6 +46,7 @@
 00486{"flow_event_id":2,"flow_event_name":"end","thread_id":0,"packet_id":1740,"source":"ps_vue.pcap","alias":"nDPId-test","flow_id":7,"flow_packet_id":2,"flow_first_seen":1568831063948,"flow_last_seen":1568831063948,"flow_tot_l4_data_len":71,"flow_min_l4_data_len":20,"flow_max_l4_data_len":51,"flow_avg_l4_data_len":35,"midstream":1,"l3_proto":"ip4","src_ip":"23.57.89.123","dst_ip":"192.168.1.132","src_port":443,"dst_port":62694,"l4_proto":"tcp","flow_datalink":1,"flow_max_packets":15}
 00523{"flow_event_id":4,"flow_event_name":"guessed","thread_id":0,"packet_id":1740,"source":"ps_vue.pcap","alias":"nDPId-test","flow_id":3,"flow_packet_id":4,"flow_first_seen":1568831058486,"flow_last_seen":1568831068662,"flow_tot_l4_data_len":1158,"flow_min_l4_data_len":20,"flow_max_l4_data_len":559,"flow_avg_l4_data_len":289,"midstream":1,"l3_proto":"ip4","src_ip":"13.33.255.96","dst_ip":"192.168.1.132","src_port":443,"dst_port":55076,"l4_proto":"tcp","ndpi": {"proto":"TLS.Amazon","breed":"Acceptable","category":"Web"}}
 00491{"flow_event_id":3,"flow_event_name":"idle","thread_id":0,"packet_id":1740,"source":"ps_vue.pcap","alias":"nDPId-test","flow_id":3,"flow_packet_id":4,"flow_first_seen":1568831058486,"flow_last_seen":1568831068662,"flow_tot_l4_data_len":1158,"flow_min_l4_data_len":20,"flow_max_l4_data_len":559,"flow_avg_l4_data_len":289,"midstream":1,"l3_proto":"ip4","src_ip":"13.33.255.96","dst_ip":"192.168.1.132","src_port":443,"dst_port":55076,"l4_proto":"tcp","flow_datalink":1,"flow_max_packets":15}
+00495{"flow_event_id":3,"flow_event_name":"idle","thread_id":0,"packet_id":1740,"source":"ps_vue.pcap","alias":"nDPId-test","flow_id":1,"flow_packet_id":255,"flow_first_seen":1568831054386,"flow_last_seen":1568831070533,"flow_tot_l4_data_len":318190,"flow_min_l4_data_len":22,"flow_max_l4_data_len":1338,"flow_avg_l4_data_len":1247,"midstream":1,"l3_proto":"ip4","src_ip":"8.252.2.139","dst_ip":"192.168.1.132","src_port":80,"dst_port":59198,"l4_proto":"tcp","flow_datalink":1,"flow_max_packets":15}
 00506{"flow_event_id":4,"flow_event_name":"guessed","thread_id":0,"packet_id":1740,"source":"ps_vue.pcap","alias":"nDPId-test","flow_id":6,"flow_packet_id":2,"flow_first_seen":1568831063828,"flow_last_seen":1568831063828,"flow_tot_l4_data_len":71,"flow_min_l4_data_len":20,"flow_max_l4_data_len":51,"flow_avg_l4_data_len":35,"midstream":1,"l3_proto":"ip4","src_ip":"23.57.89.123","dst_ip":"192.168.1.132","src_port":443,"dst_port":55648,"l4_proto":"tcp","ndpi": {"proto":"TLS","breed":"Safe","category":"Web"}}
 00486{"flow_event_id":2,"flow_event_name":"end","thread_id":0,"packet_id":1740,"source":"ps_vue.pcap","alias":"nDPId-test","flow_id":6,"flow_packet_id":2,"flow_first_seen":1568831063828,"flow_last_seen":1568831063828,"flow_tot_l4_data_len":71,"flow_min_l4_data_len":20,"flow_max_l4_data_len":51,"flow_avg_l4_data_len":35,"midstream":1,"l3_proto":"ip4","src_ip":"23.57.89.123","dst_ip":"192.168.1.132","src_port":443,"dst_port":55648,"l4_proto":"tcp","flow_datalink":1,"flow_max_packets":15}
 00509{"flow_event_id":4,"flow_event_name":"guessed","thread_id":0,"packet_id":1740,"source":"ps_vue.pcap","alias":"nDPId-test","flow_id":5,"flow_packet_id":2,"flow_first_seen":1568831063571,"flow_last_seen":1568831063589,"flow_tot_l4_data_len":723,"flow_min_l4_data_len":20,"flow_max_l4_data_len":703,"flow_avg_l4_data_len":361,"midstream":1,"l3_proto":"ip4","src_ip":"63.140.57.73","dst_ip":"192.168.1.132","src_port":443,"dst_port":61267,"l4_proto":"tcp","ndpi": {"proto":"TLS","breed":"Safe","category":"Web"}}
diff --git a/test/results/signal.pcap.out b/test/results/signal.pcap.out
index 559e7036c..951166cbd 100644
--- a/test/results/signal.pcap.out
+++ b/test/results/signal.pcap.out
@@ -276,7 +276,7 @@
 00427{"flow_id":19,"flow_packet_id":13,"packet_event_id":2,"packet_event_name":"packet-flow","thread_id":0,"packet_id":385,"source":"signal.pcap","alias":"nDPId-test","pkt_oversize":false,"pkt_ts_sec":1569051267,"pkt_ts_usec":243345,"pkt_caplen":66,"pkt_type":2048,"pkt_l3_offset":14,"pkt_l4_offset":34,"pkt_len":66,"pkt_l4_len":32,"pkt":"xiwDYGpkxGGLNYKpCABFAAA0AABAAEAGbb3AqAIRDSP9Kt7DAbsjR81wv8CYtoAQA\/8mFwAAAQEICihVno2vNN\/a"}
 00427{"flow_id":19,"flow_packet_id":14,"packet_event_id":2,"packet_event_name":"packet-flow","thread_id":0,"packet_id":386,"source":"signal.pcap","alias":"nDPId-test","pkt_oversize":false,"pkt_ts_sec":1569051267,"pkt_ts_usec":243402,"pkt_caplen":66,"pkt_type":2048,"pkt_l3_offset":14,"pkt_l4_offset":34,"pkt_len":66,"pkt_l4_len":32,"pkt":"xiwDYGpkxGGLNYKpCABFAAA0AABAAEAGbb3AqAIRDSP9Kt7DAbsjR81wv8CY+4AQA\/8l0gAAAQEICihVno2vNN\/a"}
 00500{"flow_id":19,"flow_packet_id":15,"packet_event_id":2,"packet_event_name":"packet-flow","thread_id":0,"packet_id":387,"source":"signal.pcap","alias":"nDPId-test","pkt_oversize":false,"pkt_ts_sec":1569051267,"pkt_ts_usec":250865,"pkt_caplen":119,"pkt_type":2048,"pkt_l3_offset":14,"pkt_l4_offset":34,"pkt_len":119,"pkt_l4_len":85,"pkt":"xiwDYGpkxGGLNYKpCABFAABpAABAAEAGbYjAqAIRDSP9Kt7DAbsjR81wv8CY+4AYBADbrwAAAQEICihVnpSvNN\/aFwMDADAAAAAAAAAAAVWG6O8VCZhpL3ljuuQbyjQJH99xwYFcfRw3CnAmaC4jWGNvCJKk0L4="}
-01137{"flow_event_id":5,"flow_event_name":"detected","thread_id":0,"packet_id":627,"source":"signal.pcap","alias":"nDPId-test","flow_id":19,"flow_packet_id":255,"flow_first_seen":1569051267121,"flow_last_seen":1569051267505,"flow_tot_l4_data_len":206833,"flow_min_l4_data_len":32,"flow_max_l4_data_len":1472,"flow_avg_l4_data_len":811,"midstream":0,"l3_proto":"ip4","src_ip":"192.168.2.17","dst_ip":"13.35.253.42","src_port":57027,"dst_port":443,"l4_proto":"tcp","ndpi": {"proto":"TLS.Signal","breed":"Fun","category":"Chat"},"tls": {"version":"TLSv1.2","client_requested_server_name":"cdn.signal.org","server_names":"cdn.signal.org","ja3":"6725ca90906e1036febcbfd464e2e326","ja3s":"c4b2785a87896e19d37eee932070cb22","unsafe_cipher":0,"cipher":"TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256","issuerDN":"C=US, ST=California, L=San Francisco, O=Open Whisper Systems, OU=Open Whisper Systems, CN=TextSecure","issuerDN":"C=US, ST=California, O=Open Whisper Systems, OU=Open Whisper Systems, CN=cdn.signal.org","alpn":"h2,http\/1.1","tls_supported_versions":"TLSv1.3,TLSv1.2","fingerprint":"81:3D:8A:2E:EE:B2:E1:F4:1C:2B:6D:20:16:54:B2:C1:87:D0:1E:12"}}
+01145{"flow_event_id":6,"flow_event_name":"detection-update","thread_id":0,"packet_id":627,"source":"signal.pcap","alias":"nDPId-test","flow_id":19,"flow_packet_id":255,"flow_first_seen":1569051267121,"flow_last_seen":1569051267505,"flow_tot_l4_data_len":206833,"flow_min_l4_data_len":32,"flow_max_l4_data_len":1472,"flow_avg_l4_data_len":811,"midstream":0,"l3_proto":"ip4","src_ip":"192.168.2.17","dst_ip":"13.35.253.42","src_port":57027,"dst_port":443,"l4_proto":"tcp","ndpi": {"proto":"TLS.Signal","breed":"Fun","category":"Chat"},"tls": {"version":"TLSv1.2","client_requested_server_name":"cdn.signal.org","server_names":"cdn.signal.org","ja3":"6725ca90906e1036febcbfd464e2e326","ja3s":"c4b2785a87896e19d37eee932070cb22","unsafe_cipher":0,"cipher":"TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256","issuerDN":"C=US, ST=California, L=San Francisco, O=Open Whisper Systems, OU=Open Whisper Systems, CN=TextSecure","issuerDN":"C=US, ST=California, O=Open Whisper Systems, OU=Open Whisper Systems, CN=cdn.signal.org","alpn":"h2,http\/1.1","tls_supported_versions":"TLSv1.3,TLSv1.2","fingerprint":"81:3D:8A:2E:EE:B2:E1:F4:1C:2B:6D:20:16:54:B2:C1:87:D0:1E:12"}}
 00484{"flow_event_id":3,"flow_event_name":"idle","thread_id":0,"packet_id":637,"source":"signal.pcap","alias":"nDPId-test","flow_id":1,"flow_packet_id":4,"flow_first_seen":1569051245838,"flow_last_seen":1569051261595,"flow_tot_l4_data_len":1232,"flow_min_l4_data_len":308,"flow_max_l4_data_len":308,"flow_avg_l4_data_len":308,"midstream":0,"l3_proto":"ip4","src_ip":"0.0.0.0","dst_ip":"255.255.255.255","src_port":68,"dst_port":67,"l4_proto":"udp","flow_datalink":1,"flow_max_packets":15}
 00513{"flow_event_id":4,"flow_event_name":"guessed","thread_id":0,"packet_id":637,"source":"signal.pcap","alias":"nDPId-test","flow_id":8,"flow_packet_id":8,"flow_first_seen":1569051255515,"flow_last_seen":1569051255541,"flow_tot_l4_data_len":333,"flow_min_l4_data_len":32,"flow_max_l4_data_len":78,"flow_avg_l4_data_len":41,"midstream":1,"l3_proto":"ip4","src_ip":"192.168.2.17","dst_ip":"17.248.146.144","src_port":56996,"dst_port":443,"l4_proto":"tcp","ndpi": {"proto":"TLS.Apple","breed":"Safe","category":"Web"}}
 00487{"flow_event_id":2,"flow_event_name":"end","thread_id":0,"packet_id":637,"source":"signal.pcap","alias":"nDPId-test","flow_id":8,"flow_packet_id":8,"flow_first_seen":1569051255515,"flow_last_seen":1569051255541,"flow_tot_l4_data_len":333,"flow_min_l4_data_len":32,"flow_max_l4_data_len":78,"flow_avg_l4_data_len":41,"midstream":1,"l3_proto":"ip4","src_ip":"192.168.2.17","dst_ip":"17.248.146.144","src_port":56996,"dst_port":443,"l4_proto":"tcp","flow_datalink":1,"flow_max_packets":15}
@@ -295,6 +295,7 @@
 00485{"flow_event_id":2,"flow_event_name":"end","thread_id":0,"packet_id":637,"source":"signal.pcap","alias":"nDPId-test","flow_id":9,"flow_packet_id":8,"flow_first_seen":1569051257169,"flow_last_seen":1569051257194,"flow_tot_l4_data_len":266,"flow_min_l4_data_len":20,"flow_max_l4_data_len":55,"flow_avg_l4_data_len":33,"midstream":1,"l3_proto":"ip4","src_ip":"192.168.2.17","dst_ip":"2.18.232.118","src_port":57017,"dst_port":443,"l4_proto":"tcp","flow_datalink":1,"flow_max_packets":15}
 00492{"flow_event_id":2,"flow_event_name":"end","thread_id":0,"packet_id":637,"source":"signal.pcap","alias":"nDPId-test","flow_id":3,"flow_packet_id":24,"flow_first_seen":1569051247594,"flow_last_seen":1569051257495,"flow_tot_l4_data_len":4441,"flow_min_l4_data_len":32,"flow_max_l4_data_len":1472,"flow_avg_l4_data_len":185,"midstream":0,"l3_proto":"ip4","src_ip":"192.168.2.17","dst_ip":"34.225.240.173","src_port":49226,"dst_port":443,"l4_proto":"tcp","flow_datalink":1,"flow_max_packets":15}
 00491{"flow_event_id":3,"flow_event_name":"idle","thread_id":0,"packet_id":637,"source":"signal.pcap","alias":"nDPId-test","flow_id":10,"flow_packet_id":25,"flow_first_seen":1569051264073,"flow_last_seen":1569051267100,"flow_tot_l4_data_len":5313,"flow_min_l4_data_len":32,"flow_max_l4_data_len":1472,"flow_avg_l4_data_len":212,"midstream":0,"l3_proto":"ip4","src_ip":"192.168.2.17","dst_ip":"35.169.3.40","src_port":49227,"dst_port":443,"l4_proto":"tcp","flow_datalink":1,"flow_max_packets":15}
+00495{"flow_event_id":3,"flow_event_name":"idle","thread_id":0,"packet_id":637,"source":"signal.pcap","alias":"nDPId-test","flow_id":19,"flow_packet_id":255,"flow_first_seen":1569051267121,"flow_last_seen":1569051267601,"flow_tot_l4_data_len":206833,"flow_min_l4_data_len":32,"flow_max_l4_data_len":1472,"flow_avg_l4_data_len":811,"midstream":0,"l3_proto":"ip4","src_ip":"192.168.2.17","dst_ip":"13.35.253.42","src_port":57027,"dst_port":443,"l4_proto":"tcp","flow_datalink":1,"flow_max_packets":15}
 00487{"flow_event_id":3,"flow_event_name":"idle","thread_id":0,"packet_id":637,"source":"signal.pcap","alias":"nDPId-test","flow_id":12,"flow_packet_id":2,"flow_first_seen":1569051264088,"flow_last_seen":1569051264113,"flow_tot_l4_data_len":222,"flow_min_l4_data_len":63,"flow_max_l4_data_len":159,"flow_avg_l4_data_len":111,"midstream":0,"l3_proto":"ip4","src_ip":"192.168.2.17","dst_ip":"192.168.2.1","src_port":56263,"dst_port":53,"l4_proto":"udp","flow_datalink":1,"flow_max_packets":15}
 00484{"flow_event_id":3,"flow_event_name":"idle","thread_id":0,"packet_id":637,"source":"signal.pcap","alias":"nDPId-test","flow_id":2,"flow_packet_id":2,"flow_first_seen":1569051247593,"flow_last_seen":1569051247630,"flow_tot_l4_data_len":118,"flow_min_l4_data_len":51,"flow_max_l4_data_len":67,"flow_avg_l4_data_len":59,"midstream":0,"l3_proto":"ip4","src_ip":"192.168.2.17","dst_ip":"192.168.2.1","src_port":60793,"dst_port":53,"l4_proto":"udp","flow_datalink":1,"flow_max_packets":15}
 00127{"daemon_event_id":3,"daemon_event_name":"shutdown","thread_id":0,"packet_id":637,"source":"signal.pcap","alias":"nDPId-test"}
diff --git a/test/results/skype.pcap.out b/test/results/skype.pcap.out
index 7541c53a4..d08b49ea4 100644
--- a/test/results/skype.pcap.out
+++ b/test/results/skype.pcap.out
@@ -2679,6 +2679,7 @@
 00486{"flow_event_id":3,"flow_event_name":"idle","thread_id":0,"packet_id":3284,"source":"skype.pcap","alias":"nDPId-test","flow_id":219,"flow_packet_id":2,"flow_first_seen":1431969698743,"flow_last_seen":1431969698797,"flow_tot_l4_data_len":112,"flow_min_l4_data_len":48,"flow_max_l4_data_len":64,"flow_avg_l4_data_len":56,"midstream":0,"l3_proto":"ip4","src_ip":"192.168.1.34","dst_ip":"192.168.1.1","src_port":63321,"dst_port":53,"l4_proto":"udp","flow_datalink":1,"flow_max_packets":15}
 00486{"flow_event_id":3,"flow_event_name":"idle","thread_id":0,"packet_id":3284,"source":"skype.pcap","alias":"nDPId-test","flow_id":149,"flow_packet_id":7,"flow_first_seen":1431969675950,"flow_last_seen":1431969702405,"flow_tot_l4_data_len":413,"flow_min_l4_data_len":59,"flow_max_l4_data_len":59,"flow_avg_l4_data_len":59,"midstream":0,"l3_proto":"ip4","src_ip":"192.168.1.34","dst_ip":"192.168.1.1","src_port":55159,"dst_port":53,"l4_proto":"udp","flow_datalink":1,"flow_max_packets":15}
 00484{"flow_event_id":3,"flow_event_name":"idle","thread_id":0,"packet_id":3284,"source":"skype.pcap","alias":"nDPId-test","flow_id":6,"flow_packet_id":7,"flow_first_seen":1431969642247,"flow_last_seen":1431969668794,"flow_tot_l4_data_len":273,"flow_min_l4_data_len":39,"flow_max_l4_data_len":39,"flow_avg_l4_data_len":39,"midstream":0,"l3_proto":"ip4","src_ip":"192.168.1.34","dst_ip":"192.168.1.1","src_port":65426,"dst_port":53,"l4_proto":"udp","flow_datalink":1,"flow_max_packets":15}
+00496{"flow_event_id":2,"flow_event_name":"end","thread_id":0,"packet_id":3284,"source":"skype.pcap","alias":"nDPId-test","flow_id":232,"flow_packet_id":255,"flow_first_seen":1431969710853,"flow_last_seen":1431969807279,"flow_tot_l4_data_len":88046,"flow_min_l4_data_len":32,"flow_max_l4_data_len":1472,"flow_avg_l4_data_len":345,"midstream":0,"l3_proto":"ip4","src_ip":"192.168.1.34","dst_ip":"157.56.52.28","src_port":50108,"dst_port":40009,"l4_proto":"tcp","flow_datalink":1,"flow_max_packets":15}
 00495{"flow_event_id":3,"flow_event_name":"idle","thread_id":0,"packet_id":3284,"source":"skype.pcap","alias":"nDPId-test","flow_id":233,"flow_packet_id":2,"flow_first_seen":1431969712913,"flow_last_seen":1431969712913,"flow_tot_l4_data_len":281,"flow_min_l4_data_len":140,"flow_max_l4_data_len":141,"flow_avg_l4_data_len":140,"midstream":0,"l3_proto":"ip4","src_ip":"192.168.1.34","dst_ip":"239.255.255.250","src_port":49485,"dst_port":1900,"l4_proto":"udp","flow_datalink":1,"flow_max_packets":15}
 00456{"flow_event_id":3,"flow_event_name":"idle","thread_id":0,"packet_id":3284,"source":"skype.pcap","alias":"nDPId-test","flow_id":236,"flow_packet_id":8,"flow_first_seen":1431969712918,"flow_last_seen":1431969747557,"flow_tot_l4_data_len":384,"flow_min_l4_data_len":48,"flow_max_l4_data_len":48,"flow_avg_l4_data_len":48,"midstream":0,"l3_proto":"ip4","src_ip":"192.168.1.1","dst_ip":"192.168.1.34","l4_proto":"icmp","flow_datalink":1,"flow_max_packets":15}
 00486{"flow_event_id":3,"flow_event_name":"idle","thread_id":0,"packet_id":3284,"source":"skype.pcap","alias":"nDPId-test","flow_id":276,"flow_packet_id":7,"flow_first_seen":1431969723979,"flow_last_seen":1431969750316,"flow_tot_l4_data_len":385,"flow_min_l4_data_len":55,"flow_max_l4_data_len":55,"flow_avg_l4_data_len":55,"midstream":0,"l3_proto":"ip4","src_ip":"192.168.1.34","dst_ip":"192.168.1.1","src_port":63421,"dst_port":53,"l4_proto":"udp","flow_datalink":1,"flow_max_packets":15}
diff --git a/test/results/tumblr.pcap.out b/test/results/tumblr.pcap.out
index 3a1881249..d66a9b7e9 100644
--- a/test/results/tumblr.pcap.out
+++ b/test/results/tumblr.pcap.out
@@ -434,6 +434,7 @@
 00523{"flow_event_id":3,"flow_event_name":"idle","thread_id":0,"packet_id":24745,"source":"tumblr.pcap","alias":"nDPId-test","flow_id":7,"flow_packet_id":10,"flow_first_seen":1605292104650,"flow_last_seen":1605292122733,"flow_tot_l4_data_len":762,"flow_min_l4_data_len":32,"flow_max_l4_data_len":319,"flow_avg_l4_data_len":76,"midstream":1,"l3_proto":"ip6","src_ip":"2a01:cb01:2049:8b07:991d:ec85:28df:f629","dst_ip":"64:ff9b::68f4:2ac8","src_port":56782,"dst_port":443,"l4_proto":"tcp","flow_datalink":1,"flow_max_packets":15}
 00546{"flow_event_id":4,"flow_event_name":"guessed","thread_id":0,"packet_id":24745,"source":"tumblr.pcap","alias":"nDPId-test","flow_id":37,"flow_packet_id":2,"flow_first_seen":1605292119370,"flow_last_seen":1605292119458,"flow_tot_l4_data_len":64,"flow_min_l4_data_len":32,"flow_max_l4_data_len":32,"flow_avg_l4_data_len":32,"midstream":1,"l3_proto":"ip6","src_ip":"2a01:cb01:2049:8b07:991d:ec85:28df:f629","dst_ip":"2a00:1450:4007:80b::200e","src_port":57770,"dst_port":443,"l4_proto":"tcp","ndpi": {"proto":"TLS","breed":"Safe","category":"Web"}}
 00527{"flow_event_id":3,"flow_event_name":"idle","thread_id":0,"packet_id":24745,"source":"tumblr.pcap","alias":"nDPId-test","flow_id":37,"flow_packet_id":2,"flow_first_seen":1605292119370,"flow_last_seen":1605292119458,"flow_tot_l4_data_len":64,"flow_min_l4_data_len":32,"flow_max_l4_data_len":32,"flow_avg_l4_data_len":32,"midstream":1,"l3_proto":"ip6","src_ip":"2a01:cb01:2049:8b07:991d:ec85:28df:f629","dst_ip":"2a00:1450:4007:80b::200e","src_port":57770,"dst_port":443,"l4_proto":"tcp","flow_datalink":1,"flow_max_packets":15}
+00530{"flow_event_id":3,"flow_event_name":"idle","thread_id":0,"packet_id":24745,"source":"tumblr.pcap","alias":"nDPId-test","flow_id":14,"flow_packet_id":255,"flow_first_seen":1605292105669,"flow_last_seen":1605292122890,"flow_tot_l4_data_len":153848,"flow_min_l4_data_len":32,"flow_max_l4_data_len":1432,"flow_avg_l4_data_len":603,"midstream":1,"l3_proto":"ip6","src_ip":"2a01:cb01:2049:8b07:991d:ec85:28df:f629","dst_ip":"64:ff9b::c000:4d03","src_port":56794,"dst_port":443,"l4_proto":"tcp","flow_datalink":1,"flow_max_packets":15}
 00546{"flow_event_id":4,"flow_event_name":"guessed","thread_id":0,"packet_id":24745,"source":"tumblr.pcap","alias":"nDPId-test","flow_id":29,"flow_packet_id":2,"flow_first_seen":1605292116554,"flow_last_seen":1605292116783,"flow_tot_l4_data_len":64,"flow_min_l4_data_len":32,"flow_max_l4_data_len":32,"flow_avg_l4_data_len":32,"midstream":1,"l3_proto":"ip6","src_ip":"2a01:cb01:2049:8b07:991d:ec85:28df:f629","dst_ip":"2a00:1450:4007:80b::200e","src_port":57788,"dst_port":443,"l4_proto":"tcp","ndpi": {"proto":"TLS","breed":"Safe","category":"Web"}}
 00527{"flow_event_id":3,"flow_event_name":"idle","thread_id":0,"packet_id":24745,"source":"tumblr.pcap","alias":"nDPId-test","flow_id":29,"flow_packet_id":2,"flow_first_seen":1605292116554,"flow_last_seen":1605292116783,"flow_tot_l4_data_len":64,"flow_min_l4_data_len":32,"flow_max_l4_data_len":32,"flow_avg_l4_data_len":32,"midstream":1,"l3_proto":"ip6","src_ip":"2a01:cb01:2049:8b07:991d:ec85:28df:f629","dst_ip":"2a00:1450:4007:80b::200e","src_port":57788,"dst_port":443,"l4_proto":"tcp","flow_datalink":1,"flow_max_packets":15}
 00532{"flow_event_id":3,"flow_event_name":"idle","thread_id":0,"packet_id":24745,"source":"tumblr.pcap","alias":"nDPId-test","flow_id":20,"flow_packet_id":1671,"flow_first_seen":1605292108895,"flow_last_seen":1605292115212,"flow_tot_l4_data_len":1089230,"flow_min_l4_data_len":32,"flow_max_l4_data_len":2458,"flow_avg_l4_data_len":651,"midstream":0,"l3_proto":"ip6","src_ip":"2a01:cb01:2049:8b07:991d:ec85:28df:f629","dst_ip":"64:ff9b::c000:4d03","src_port":56842,"dst_port":443,"l4_proto":"tcp","flow_datalink":1,"flow_max_packets":15}
@@ -445,6 +446,7 @@
 00527{"flow_event_id":3,"flow_event_name":"idle","thread_id":0,"packet_id":24745,"source":"tumblr.pcap","alias":"nDPId-test","flow_id":38,"flow_packet_id":2,"flow_first_seen":1605292120654,"flow_last_seen":1605292120839,"flow_tot_l4_data_len":64,"flow_min_l4_data_len":32,"flow_max_l4_data_len":32,"flow_avg_l4_data_len":32,"midstream":1,"l3_proto":"ip6","src_ip":"2a01:cb01:2049:8b07:991d:ec85:28df:f629","dst_ip":"2a00:1450:4007:808::200e","src_port":58004,"dst_port":443,"l4_proto":"tcp","flow_datalink":1,"flow_max_packets":15}
 00540{"flow_event_id":4,"flow_event_name":"guessed","thread_id":0,"packet_id":24745,"source":"tumblr.pcap","alias":"nDPId-test","flow_id":35,"flow_packet_id":2,"flow_first_seen":1605292118602,"flow_last_seen":1605292118777,"flow_tot_l4_data_len":64,"flow_min_l4_data_len":32,"flow_max_l4_data_len":32,"flow_avg_l4_data_len":32,"midstream":1,"l3_proto":"ip6","src_ip":"2a01:cb01:2049:8b07:991d:ec85:28df:f629","dst_ip":"64:ff9b::d83a:d582","src_port":50906,"dst_port":443,"l4_proto":"tcp","ndpi": {"proto":"TLS","breed":"Safe","category":"Web"}}
 00521{"flow_event_id":3,"flow_event_name":"idle","thread_id":0,"packet_id":24745,"source":"tumblr.pcap","alias":"nDPId-test","flow_id":35,"flow_packet_id":2,"flow_first_seen":1605292118602,"flow_last_seen":1605292118777,"flow_tot_l4_data_len":64,"flow_min_l4_data_len":32,"flow_max_l4_data_len":32,"flow_avg_l4_data_len":32,"midstream":1,"l3_proto":"ip6","src_ip":"2a01:cb01:2049:8b07:991d:ec85:28df:f629","dst_ip":"64:ff9b::d83a:d582","src_port":50906,"dst_port":443,"l4_proto":"tcp","flow_datalink":1,"flow_max_packets":15}
+00529{"flow_event_id":3,"flow_event_name":"idle","thread_id":0,"packet_id":24745,"source":"tumblr.pcap","alias":"nDPId-test","flow_id":6,"flow_packet_id":255,"flow_first_seen":1605292103810,"flow_last_seen":1605292122755,"flow_tot_l4_data_len":152530,"flow_min_l4_data_len":32,"flow_max_l4_data_len":1432,"flow_avg_l4_data_len":598,"midstream":1,"l3_proto":"ip6","src_ip":"2a01:cb01:2049:8b07:991d:ec85:28df:f629","dst_ip":"64:ff9b::98c7:1593","src_port":42908,"dst_port":443,"l4_proto":"tcp","flow_datalink":1,"flow_max_packets":15}
 00543{"flow_event_id":4,"flow_event_name":"guessed","thread_id":0,"packet_id":24745,"source":"tumblr.pcap","alias":"nDPId-test","flow_id":5,"flow_packet_id":9,"flow_first_seen":1605292103804,"flow_last_seen":1605292104007,"flow_tot_l4_data_len":1576,"flow_min_l4_data_len":32,"flow_max_l4_data_len":676,"flow_avg_l4_data_len":175,"midstream":1,"l3_proto":"ip6","src_ip":"2a01:cb01:2049:8b07:991d:ec85:28df:f629","dst_ip":"64:ff9b::8fcc:d927","src_port":57286,"dst_port":443,"l4_proto":"tcp","ndpi": {"proto":"TLS","breed":"Safe","category":"Web"}}
 00524{"flow_event_id":3,"flow_event_name":"idle","thread_id":0,"packet_id":24745,"source":"tumblr.pcap","alias":"nDPId-test","flow_id":5,"flow_packet_id":9,"flow_first_seen":1605292103804,"flow_last_seen":1605292104007,"flow_tot_l4_data_len":1576,"flow_min_l4_data_len":32,"flow_max_l4_data_len":676,"flow_avg_l4_data_len":175,"midstream":1,"l3_proto":"ip6","src_ip":"2a01:cb01:2049:8b07:991d:ec85:28df:f629","dst_ip":"64:ff9b::8fcc:d927","src_port":57286,"dst_port":443,"l4_proto":"tcp","flow_datalink":1,"flow_max_packets":15}
 00547{"flow_event_id":3,"flow_event_name":"idle","thread_id":0,"packet_id":24745,"source":"tumblr.pcap","alias":"nDPId-test","flow_id":10,"flow_packet_id":50,"flow_first_seen":1605292105197,"flow_last_seen":1605292105378,"flow_tot_l4_data_len":21326,"flow_min_l4_data_len":32,"flow_max_l4_data_len":1240,"flow_avg_l4_data_len":426,"midstream":0,"l3_proto":"ip6","src_ip":"2a01:cb01:2049:8b07:991d:ec85:28df:f629","dst_ip":"2606:2800:135:155a:23ba:b2a:25ff:122d","src_port":58380,"dst_port":443,"l4_proto":"tcp","flow_datalink":1,"flow_max_packets":15}
@@ -464,6 +466,8 @@
 00546{"flow_event_id":4,"flow_event_name":"guessed","thread_id":0,"packet_id":24745,"source":"tumblr.pcap","alias":"nDPId-test","flow_id":48,"flow_packet_id":2,"flow_first_seen":1605292122874,"flow_last_seen":1605292122899,"flow_tot_l4_data_len":64,"flow_min_l4_data_len":32,"flow_max_l4_data_len":32,"flow_avg_l4_data_len":32,"midstream":1,"l3_proto":"ip6","src_ip":"2a01:cb01:2049:8b07:991d:ec85:28df:f629","dst_ip":"2a00:1450:4007:80a::200a","src_port":40190,"dst_port":443,"l4_proto":"tcp","ndpi": {"proto":"TLS","breed":"Safe","category":"Web"}}
 00527{"flow_event_id":3,"flow_event_name":"idle","thread_id":0,"packet_id":24745,"source":"tumblr.pcap","alias":"nDPId-test","flow_id":48,"flow_packet_id":2,"flow_first_seen":1605292122874,"flow_last_seen":1605292122899,"flow_tot_l4_data_len":64,"flow_min_l4_data_len":32,"flow_max_l4_data_len":32,"flow_avg_l4_data_len":32,"midstream":1,"l3_proto":"ip6","src_ip":"2a01:cb01:2049:8b07:991d:ec85:28df:f629","dst_ip":"2a00:1450:4007:80a::200a","src_port":40190,"dst_port":443,"l4_proto":"tcp","flow_datalink":1,"flow_max_packets":15}
 00528{"flow_event_id":3,"flow_event_name":"idle","thread_id":0,"packet_id":24745,"source":"tumblr.pcap","alias":"nDPId-test","flow_id":41,"flow_packet_id":62,"flow_first_seen":1605292121486,"flow_last_seen":1605292122503,"flow_tot_l4_data_len":26052,"flow_min_l4_data_len":32,"flow_max_l4_data_len":1432,"flow_avg_l4_data_len":420,"midstream":0,"l3_proto":"ip6","src_ip":"2a01:cb01:2049:8b07:991d:ec85:28df:f629","dst_ip":"64:ff9b::4a72:9a16","src_port":43328,"dst_port":443,"l4_proto":"tcp","flow_datalink":1,"flow_max_packets":15}
+00529{"flow_event_id":3,"flow_event_name":"idle","thread_id":0,"packet_id":24745,"source":"tumblr.pcap","alias":"nDPId-test","flow_id":8,"flow_packet_id":255,"flow_first_seen":1605292105170,"flow_last_seen":1605292122449,"flow_tot_l4_data_len":162300,"flow_min_l4_data_len":32,"flow_max_l4_data_len":1432,"flow_avg_l4_data_len":636,"midstream":1,"l3_proto":"ip6","src_ip":"2a01:cb01:2049:8b07:991d:ec85:28df:f629","dst_ip":"64:ff9b::c000:4d28","src_port":43420,"dst_port":443,"l4_proto":"tcp","flow_datalink":1,"flow_max_packets":15}
+00529{"flow_event_id":3,"flow_event_name":"idle","thread_id":0,"packet_id":24745,"source":"tumblr.pcap","alias":"nDPId-test","flow_id":9,"flow_packet_id":255,"flow_first_seen":1605292105171,"flow_last_seen":1605292122739,"flow_tot_l4_data_len":155628,"flow_min_l4_data_len":32,"flow_max_l4_data_len":1432,"flow_avg_l4_data_len":610,"midstream":1,"l3_proto":"ip6","src_ip":"2a01:cb01:2049:8b07:991d:ec85:28df:f629","dst_ip":"64:ff9b::c000:4d28","src_port":43434,"dst_port":443,"l4_proto":"tcp","flow_datalink":1,"flow_max_packets":15}
 00539{"flow_event_id":4,"flow_event_name":"guessed","thread_id":0,"packet_id":24745,"source":"tumblr.pcap","alias":"nDPId-test","flow_id":24,"flow_packet_id":2,"flow_first_seen":1605292116554,"flow_last_seen":1605292116783,"flow_tot_l4_data_len":64,"flow_min_l4_data_len":32,"flow_max_l4_data_len":32,"flow_avg_l4_data_len":32,"midstream":1,"l3_proto":"ip6","src_ip":"2a01:cb01:2049:8b07:991d:ec85:28df:f629","dst_ip":"64:ff9b::df9:21c6","src_port":43602,"dst_port":443,"l4_proto":"tcp","ndpi": {"proto":"TLS","breed":"Safe","category":"Web"}}
 00520{"flow_event_id":3,"flow_event_name":"idle","thread_id":0,"packet_id":24745,"source":"tumblr.pcap","alias":"nDPId-test","flow_id":24,"flow_packet_id":2,"flow_first_seen":1605292116554,"flow_last_seen":1605292116783,"flow_tot_l4_data_len":64,"flow_min_l4_data_len":32,"flow_max_l4_data_len":32,"flow_avg_l4_data_len":32,"midstream":1,"l3_proto":"ip6","src_ip":"2a01:cb01:2049:8b07:991d:ec85:28df:f629","dst_ip":"64:ff9b::df9:21c6","src_port":43602,"dst_port":443,"l4_proto":"tcp","flow_datalink":1,"flow_max_packets":15}
 00544{"flow_event_id":4,"flow_event_name":"guessed","thread_id":0,"packet_id":24745,"source":"tumblr.pcap","alias":"nDPId-test","flow_id":15,"flow_packet_id":28,"flow_first_seen":1605292105726,"flow_last_seen":1605292122804,"flow_tot_l4_data_len":2267,"flow_min_l4_data_len":32,"flow_max_l4_data_len":189,"flow_avg_l4_data_len":80,"midstream":1,"l3_proto":"ip6","src_ip":"2a01:cb01:2049:8b07:991d:ec85:28df:f629","dst_ip":"64:ff9b::c000:4c03","src_port":51874,"dst_port":443,"l4_proto":"tcp","ndpi": {"proto":"TLS","breed":"Safe","category":"Web"}}
diff --git a/test/results/wa_voice.pcap.out b/test/results/wa_voice.pcap.out
index 99b00074f..73e780ba7 100644
--- a/test/results/wa_voice.pcap.out
+++ b/test/results/wa_voice.pcap.out
@@ -281,6 +281,7 @@
 00496{"flow_event_id":3,"flow_event_name":"idle","thread_id":0,"packet_id":736,"source":"wa_voice.pcap","alias":"nDPId-test","flow_id":22,"flow_packet_id":3,"flow_first_seen":1561455709984,"flow_last_seen":1561455716020,"flow_tot_l4_data_len":390,"flow_min_l4_data_len":109,"flow_max_l4_data_len":141,"flow_avg_l4_data_len":130,"midstream":0,"l3_proto":"ip4","src_ip":"192.168.2.12","dst_ip":"239.255.255.250","src_port":64716,"dst_port":1900,"l4_proto":"udp","flow_datalink":1,"flow_max_packets":15}
 00458{"flow_event_id":3,"flow_event_name":"idle","thread_id":0,"packet_id":736,"source":"wa_voice.pcap","alias":"nDPId-test","flow_id":30,"flow_packet_id":4,"flow_first_seen":1561455741484,"flow_last_seen":1561455742405,"flow_tot_l4_data_len":144,"flow_min_l4_data_len":36,"flow_max_l4_data_len":36,"flow_avg_l4_data_len":36,"midstream":0,"l3_proto":"ip4","src_ip":"192.168.2.12","dst_ip":"91.252.56.51","l4_proto":"icmp","flow_datalink":1,"flow_max_packets":15}
 00487{"flow_event_id":3,"flow_event_name":"idle","thread_id":0,"packet_id":736,"source":"wa_voice.pcap","alias":"nDPId-test","flow_id":20,"flow_packet_id":2,"flow_first_seen":1561455707435,"flow_last_seen":1561455707470,"flow_tot_l4_data_len":125,"flow_min_l4_data_len":42,"flow_max_l4_data_len":83,"flow_avg_l4_data_len":62,"midstream":0,"l3_proto":"ip4","src_ip":"192.168.2.12","dst_ip":"192.168.2.1","src_port":60549,"dst_port":53,"l4_proto":"udp","flow_datalink":1,"flow_max_packets":15}
+00497{"flow_event_id":3,"flow_event_name":"idle","thread_id":0,"packet_id":736,"source":"wa_voice.pcap","alias":"nDPId-test","flow_id":5,"flow_packet_id":255,"flow_first_seen":1561455688704,"flow_last_seen":1561455743434,"flow_tot_l4_data_len":29222,"flow_min_l4_data_len":32,"flow_max_l4_data_len":1420,"flow_avg_l4_data_len":114,"midstream":0,"l3_proto":"ip4","src_ip":"192.168.2.12","dst_ip":"157.240.20.53","src_port":49355,"dst_port":5222,"l4_proto":"tcp","flow_datalink":1,"flow_max_packets":15}
 00493{"flow_event_id":3,"flow_event_name":"idle","thread_id":0,"packet_id":736,"source":"wa_voice.pcap","alias":"nDPId-test","flow_id":17,"flow_packet_id":8,"flow_first_seen":1561455706913,"flow_last_seen":1561455741420,"flow_tot_l4_data_len":826,"flow_min_l4_data_len":52,"flow_max_l4_data_len":134,"flow_avg_l4_data_len":103,"midstream":0,"l3_proto":"ip4","src_ip":"192.168.2.12","dst_ip":"179.60.192.48","src_port":56328,"dst_port":3478,"l4_proto":"udp","flow_datalink":1,"flow_max_packets":15}
 00486{"flow_event_id":3,"flow_event_name":"idle","thread_id":0,"packet_id":736,"source":"wa_voice.pcap","alias":"nDPId-test","flow_id":2,"flow_packet_id":2,"flow_first_seen":1561455687991,"flow_last_seen":1561455688018,"flow_tot_l4_data_len":119,"flow_min_l4_data_len":40,"flow_max_l4_data_len":79,"flow_avg_l4_data_len":59,"midstream":0,"l3_proto":"ip4","src_ip":"192.168.2.12","dst_ip":"192.168.2.1","src_port":60765,"dst_port":53,"l4_proto":"udp","flow_datalink":1,"flow_max_packets":15}
 00493{"flow_event_id":3,"flow_event_name":"idle","thread_id":0,"packet_id":736,"source":"wa_voice.pcap","alias":"nDPId-test","flow_id":15,"flow_packet_id":8,"flow_first_seen":1561455706912,"flow_last_seen":1561455741419,"flow_tot_l4_data_len":826,"flow_min_l4_data_len":52,"flow_max_l4_data_len":134,"flow_avg_l4_data_len":103,"midstream":0,"l3_proto":"ip4","src_ip":"192.168.2.12","dst_ip":"185.60.216.51","src_port":56328,"dst_port":3478,"l4_proto":"udp","flow_datalink":1,"flow_max_packets":15}
diff --git a/test/run_tests.sh b/test/run_tests.sh
index ef7a15243..5c6e54c19 100755
--- a/test/run_tests.sh
+++ b/test/run_tests.sh
@@ -5,14 +5,18 @@ set -e
 LINE_SPACES=${LINE_SPACES:-48}
 MYDIR="$(realpath "$(dirname ${0})")"
 nDPId_test_EXEC="${2:-"$(realpath "${MYDIR}/../nDPId-test")"}"
+NETCAT_EXEC="nc -q 0 -l 127.0.0.1 9000"
 JSON_VALIDATOR="${3:-"$(realpath "${MYDIR}/../examples/py-schema-validation/py-schema-validation.py")"}"
+SEMN_VALIDATOR="${3:-"$(realpath "${MYDIR}/../examples/py-semantic-validation/py-semantic-validation.py")"}"
 
-if [ $# -ne 1 -a $# -ne 2 -a $# -ne 3 ]; then
+if [ $# -ne 1 -a $# -ne 2 -a $# -ne 3 -a $# -ne 4 ]; then
 cat <<EOF
-usage: ${0} [path-to-nDPI-source-root] [path-to-nDPId-test-exec] [path-to-nDPId-JSON-validator]
+usage: ${0} [path-to-nDPI-source-root] \\
+	[path-to-nDPId-test-exec] [path-to-nDPId-JSON-validator] [path-to-nDPId-SEMANTIC-validator]
 
-	path-to-nDPId-test-exec defaults to ${nDPId_test_EXEC}
-	path-to-nDPId-JSON-validator defaults to ${JSON_VALIDATOR}
+	path-to-nDPId-test-exec defaults to         ${nDPId_test_EXEC}
+	path-to-nDPId-JSON-validator defaults to    ${JSON_VALIDATOR}
+	path-to-nDPId-SEMANTIC-validator default to ${SEMN_VALIDATOR}
 EOF
 exit 2
 fi
@@ -64,9 +68,17 @@ mkdir -p /tmp/nDPId-test-stderr
 set +e
 TESTS_FAILED=0
 
-for pcap_file in $(ls *.pcap*); do
-    printf '%s\n' "${nDPId_test_EXEC} ${pcap_file}" \
+for pcap_file in $(ls *.pcap *.pcapng *.cap); do
+    if file "${pcap_file}" | grep -qoE ':\s(pcap|pcap-ng) capture file'; then
+        true # pass
+    else
+        continue
+    fi
+
+    printf '%s\n' "-- CMD: ${nDPId_test_EXEC} $(realpath "${pcap_file}")" \
         >"/tmp/nDPId-test-stderr/${pcap_file}.out"
+    printf '%s\n' "-- OUT: ${MYDIR}/results/${pcap_file}.out" \
+        >>"/tmp/nDPId-test-stderr/${pcap_file}.out"
 
     ${nDPId_test_EXEC} "${pcap_file}" \
         >"${MYDIR}/results/${pcap_file}.out.new" \
@@ -94,7 +106,7 @@ for pcap_file in $(ls *.pcap*); do
     else
         printf '%s\n' '[FAIL]'
         printf '%s\n' '----------------------------------------'
-        printf '%s\n' "-- STDERR of ${pcap_file}"
+        printf '%s\n' "-- STDERR of ${pcap_file}: /tmp/nDPId-test-stderr/${pcap_file}.out"
         cat "/tmp/nDPId-test-stderr/${pcap_file}.out"
         TESTS_FAILED=$((TESTS_FAILED + 1))
     fi
@@ -102,11 +114,47 @@ for pcap_file in $(ls *.pcap*); do
     rm -f "${MYDIR}/results/${pcap_file}.out.new"
 done
 
+function validate_results()
+{
+    prefix_str="${1}"
+    pcap_file="$(basename ${2})"
+    result_file="${3}"
+    validator_exec="${4}"
+
+    printf "${prefix_str} %-$((${LINE_SPACES} - ${#prefix_str}))s\t" "${pcap_file}"
+    printf '%s\n' "-- ${prefix_str}" >>"/tmp/nDPId-test-stderr/${pcap_file}.out"
+
+    if [ ! -r "${result_file}" ]; then
+        printf ' %s\n' '[MISSING]'
+        return 1
+    fi
+
+    cat "${result_file}" | ${NETCAT_EXEC} &
+    nc_pid=$!
+    printf '%s\n' "-- ${validator_exec}" >>"/tmp/nDPId-test-stderr/${pcap_file}.out"
+    ${validator_exec} 2>>"/tmp/nDPId-test-stderr/${pcap_file}.out"
+    if [ $? -eq 0 ]; then
+        printf ' %s\n' '[OK]'
+    else
+        printf ' %s\n' '[FAIL]'
+        printf '%s\n' '----------------------------------------'
+        printf '%s\n' "-- STDERR of ${pcap_file}: /tmp/nDPId-test-stderr/${pcap_file}.out"
+        cat "/tmp/nDPId-test-stderr/${pcap_file}.out"
+        return 1
+    fi
+    kill -SIGTERM ${nc_pid} 2>/dev/null
+    wait ${nc_pid} 2>/dev/null
+
+    return 0
+}
+
 cat <<EOF
 
-----------------------------
--- JSON schema validation --
-----------------------------
+--------------------------------
+-- SCHEMA/SEMANTIC Validation --
+--------------------------------
+
+netcat (OpenBSD) exec + args: ${NETCAT_EXEC}
 
 EOF
 
@@ -117,28 +165,19 @@ for out_file in $(ls results/*.out); do
         printf "%-${LINE_SPACES}s\t%s\n" "$(basename ${pcap_file})" '[MISSING]'
         TESTS_FAILED=$((TESTS_FAILED + 1))
     else
-        printf "SCHEMA %-${LINE_SPACES}s\t" "$(basename ${pcap_file})"
-        printf '%s\n' '*** JSON schema validation ***' >>"/tmp/nDPId-test-stderr/$(basename ${pcap_file}).out"
-        if [ ! -r "${out_file}" ]; then
-            printf ' %s\n' '[MISSING]'
+        validate_results "SCHEMA  " "${pcap_file}" "${out_file}" \
+            "${JSON_VALIDATOR} --host 127.0.0.1 --port 9000"
+        if [ $? -ne 0 ]; then
             TESTS_FAILED=$((TESTS_FAILED + 1))
             continue
         fi
-        cat "${out_file}" | nc -q 1 -l 127.0.0.1 9000 &
-        nc_pid=$!
-        ${JSON_VALIDATOR} \
-            --host 127.0.0.1 --port 9000 2>>"/tmp/nDPId-test-stderr/$(basename ${pcap_file}).out"
-        if [ $? -eq 0 ]; then
-            printf ' %s\n' '[OK]'
-        else
-            printf ' %s\n' '[FAIL]'
-            printf '%s\n' '----------------------------------------'
-            printf '%s\n' "-- STDERR of $(basename ${pcap_file})"
-            cat "/tmp/nDPId-test-stderr/$(basename ${pcap_file}).out"
+
+        validate_results "SEMANTIC" "${pcap_file}" "${out_file}" \
+            "${SEMN_VALIDATOR} --host 127.0.0.1 --port 9000 --strict"
+        if [ $? -ne 0 ]; then
             TESTS_FAILED=$((TESTS_FAILED + 1))
+            continue
         fi
-        kill -SIGTERM ${nc_pid} 2>/dev/null
-        wait ${nc_pid} 2>/dev/null
     fi
 done