nDPId: Reduced flow-updates for TCP flows to 1/4 of the timeout value.

 * nDPId: Fixed broken validation tests.
 * nDPId: Removed TICK_RESOLUTION, not required anymore.
 * c-collectd: Improved total layer4 payload calculation/update handling.
 * c-collectd: Updated RRD Graph script according to total layer4 payload changes.
 * py-flow-info.py: Fixed several bugs and syntax errors.
 * Python scripts: Added dirname(argv[0]) as search path for nDPIsrvd.py.
 * nDPIsrvd&nDPId-test: Fixed missing EPOLLERR check.

Signed-off-by: Toni Uhlig <matzeton@googlemail.com>

11 files changed, 153 insertions, 99 deletions

diff --git a/examples/c-collectd/c-collectd.c b/examples/c-collectd/c-collectd.c
index 29b060e60..bd880d45c 100644
--- a/examples/c-collectd/c-collectd.c
+++ b/examples/c-collectd/c-collectd.c
@@ -24,6 +24,12 @@
         syslog(flags, format, __VA_ARGS__);                                                                            \
     }
 
+struct flow_user_data
+{
+    nDPIsrvd_ull last_flow_src_l4_payload_len;
+    nDPIsrvd_ull last_flow_dst_l4_payload_len;
+};
+
 static int main_thread_shutdown = 0;
 static int collectd_timerfd = -1;
 static pid_t collectd_pid;
@@ -45,7 +51,8 @@ static struct
     uint64_t flow_detection_update_count;
     uint64_t flow_not_detected_count;
 
-    uint64_t flow_total_bytes;
+    uint64_t flow_src_total_bytes;
+    uint64_t flow_dst_total_bytes;
     uint64_t flow_risky_count;
 
     uint64_t flow_breed_safe_count;
@@ -264,8 +271,8 @@ static void print_collectd_exec_output(void)
     printf(COLLECTD_PUTVAL_N_FORMAT(flow_new_count) COLLECTD_PUTVAL_N_FORMAT(flow_end_count)
                COLLECTD_PUTVAL_N_FORMAT(flow_idle_count) COLLECTD_PUTVAL_N_FORMAT(flow_guessed_count)
                    COLLECTD_PUTVAL_N_FORMAT(flow_detected_count) COLLECTD_PUTVAL_N_FORMAT(flow_detection_update_count)
-                       COLLECTD_PUTVAL_N_FORMAT(flow_not_detected_count) COLLECTD_PUTVAL_N_FORMAT(flow_total_bytes)
-                           COLLECTD_PUTVAL_N_FORMAT(flow_risky_count),
+                       COLLECTD_PUTVAL_N_FORMAT(flow_not_detected_count) COLLECTD_PUTVAL_N_FORMAT(flow_src_total_bytes)
+                           COLLECTD_PUTVAL_N_FORMAT(flow_dst_total_bytes) COLLECTD_PUTVAL_N_FORMAT(flow_risky_count),
 
            COLLECTD_PUTVAL_N(flow_new_count),
            COLLECTD_PUTVAL_N(flow_end_count),
@@ -274,7 +281,8 @@ static void print_collectd_exec_output(void)
            COLLECTD_PUTVAL_N(flow_detected_count),
            COLLECTD_PUTVAL_N(flow_detection_update_count),
            COLLECTD_PUTVAL_N(flow_not_detected_count),
-           COLLECTD_PUTVAL_N(flow_total_bytes),
+           COLLECTD_PUTVAL_N(flow_src_total_bytes),
+           COLLECTD_PUTVAL_N(flow_dst_total_bytes),
            COLLECTD_PUTVAL_N(flow_risky_count));
 
     printf(COLLECTD_PUTVAL_N_FORMAT(flow_breed_safe_count) COLLECTD_PUTVAL_N_FORMAT(flow_breed_acceptable_count)
@@ -431,21 +439,6 @@ static int mainloop(int epollfd, struct nDPIsrvd_socket * const sock)
     return 0;
 }
 
-static uint64_t get_total_flow_bytes(struct nDPIsrvd_socket * const sock)
-{
-    nDPIsrvd_ull total_bytes_ull[2] = {0, 0};
-
-    if (TOKEN_VALUE_TO_ULL(TOKEN_GET_SZ(sock, "flow_src_tot_l4_payload_len"), &total_bytes_ull[0]) == CONVERSION_OK &&
-        TOKEN_VALUE_TO_ULL(TOKEN_GET_SZ(sock, "flow_dst_tot_l4_payload_len"), &total_bytes_ull[1]) == CONVERSION_OK)
-    {
-        return total_bytes_ull[0] + total_bytes_ull[1];
-    }
-    else
-    {
-        return 0;
-    }
-}
-
 static enum nDPIsrvd_callback_return captured_json_callback(struct nDPIsrvd_socket * const sock,
                                                             struct nDPIsrvd_instance * const instance,
                                                             struct nDPIsrvd_thread_data * const thread_data,
@@ -457,6 +450,25 @@ static enum nDPIsrvd_callback_return captured_json_callback(struct nDPIsrvd_sock
     (void)flow;
 
     struct nDPIsrvd_json_token const * const flow_event_name = TOKEN_GET_SZ(sock, "flow_event_name");
+    struct flow_user_data * const flow_user_data = (struct flow_user_data *)flow->flow_user_data;
+
+    if (flow_user_data != NULL)
+    {
+        nDPIsrvd_ull total_bytes_ull[2] = {0, 0};
+
+        if (TOKEN_VALUE_TO_ULL(TOKEN_GET_SZ(sock, "flow_src_tot_l4_payload_len"), &total_bytes_ull[0]) ==
+                CONVERSION_OK &&
+            TOKEN_VALUE_TO_ULL(TOKEN_GET_SZ(sock, "flow_dst_tot_l4_payload_len"), &total_bytes_ull[1]) == CONVERSION_OK)
+        {
+            collectd_statistics.flow_src_total_bytes +=
+                total_bytes_ull[0] - flow_user_data->last_flow_src_l4_payload_len;
+            collectd_statistics.flow_dst_total_bytes +=
+                total_bytes_ull[1] - flow_user_data->last_flow_dst_l4_payload_len;
+
+            flow_user_data->last_flow_src_l4_payload_len = total_bytes_ull[0];
+            flow_user_data->last_flow_dst_l4_payload_len = total_bytes_ull[1];
+        }
+    }
 
     if (TOKEN_VALUE_EQUALS_SZ(flow_event_name, "new") != 0)
     {
@@ -497,12 +509,10 @@ static enum nDPIsrvd_callback_return captured_json_callback(struct nDPIsrvd_sock
     else if (TOKEN_VALUE_EQUALS_SZ(flow_event_name, "end") != 0)
     {
         collectd_statistics.flow_end_count++;
-        collectd_statistics.flow_total_bytes += get_total_flow_bytes(sock);
     }
     else if (TOKEN_VALUE_EQUALS_SZ(flow_event_name, "idle") != 0)
     {
         collectd_statistics.flow_idle_count++;
-        collectd_statistics.flow_total_bytes += get_total_flow_bytes(sock);
     }
     else if (TOKEN_VALUE_EQUALS_SZ(flow_event_name, "guessed") != 0)
     {
@@ -687,7 +697,8 @@ int main(int argc, char ** argv)
 
     openlog("nDPIsrvd-collectd", LOG_CONS, LOG_DAEMON);
 
-    struct nDPIsrvd_socket * sock = nDPIsrvd_socket_init(0, 0, 0, 0, captured_json_callback, NULL, NULL);
+    struct nDPIsrvd_socket * sock =
+        nDPIsrvd_socket_init(0, 0, 0, sizeof(struct flow_user_data), captured_json_callback, NULL, NULL);
     if (sock == NULL)
     {
         LOG(LOG_DAEMON | LOG_ERR, "%s", "nDPIsrvd socket memory allocation failed!");
diff --git a/examples/c-collectd/rrdgraph.sh b/examples/c-collectd/rrdgraph.sh
index a4dc298fe..6690bf9d5 100755
--- a/examples/c-collectd/rrdgraph.sh
+++ b/examples/c-collectd/rrdgraph.sh
@@ -2,7 +2,7 @@
 
 RRDDIR="${1}"
 OUTDIR="${2}"
-RRDARGS="--width=800 --height=400 -v Amount"
+RRDARGS="--width=800 --height=400"
 
 if [ -z "${RRDDIR}" ]; then
 	printf '%s: Missing RRD directory which contains nDPIsrvd/Collectd files.\n' "${0}"
@@ -14,7 +14,7 @@ if [ -z "${OUTDIR}" ]; then
 	exit 1
 fi
 
-if [ $(ls -al ${RRDDIR}/flow_*.rrd | wc -l) -ne 54 ]; then
+if [ $(ls -al ${RRDDIR}/gauge-flow_*.rrd | wc -l) -ne 54 ]; then
 	printf '%s: Missing some *.rrd files.\n' "${0}"
 	exit 1
 fi
@@ -25,6 +25,7 @@ if [ ! -r "${OUTDIR}/index.html" -o ! -r "${OUTDIR}/flows.html" -o ! -r "${OUTDI
 fi
 
 TIME_PAST_HOUR="--start=-3600 --end=-0"
+TIME_PAST_12HOURS="--start=-43200 --end=-0"
 TIME_PAST_DAY="--start=-86400 --end=-0"
 TIME_PAST_WEEK="--start=-604800 --end=-0"
 TIME_PAST_MONTH="--start=-2419200 --end=-0"
@@ -44,17 +45,20 @@ rrdtool_graph_print_cur_min_max_avg() {
 rrdtool_graph() {
 	TITLE="${1}"
 	shift
+	YAXIS_NAME="${1}"
+	shift
 	OUTPNG="${1}"
 	shift
 
-	rrdtool graph ${RRDARGS} -t "${TITLE} (past hour)"  -Y --start=-3600     --end=-0 "${OUTPNG}_past_hour.png"  ${*}
-	rrdtool graph ${RRDARGS} -t "${TITLE} (past day)"   -Y --start=-86400    --end=-0 "${OUTPNG}_past_day.png"   ${*}
-	rrdtool graph ${RRDARGS} -t "${TITLE} (past week)"  -Y --start=-604800   --end=-0 "${OUTPNG}_past_week.png"  ${*}
-	rrdtool graph ${RRDARGS} -t "${TITLE} (past month)" -Y --start=-2419200  --end=-0 "${OUTPNG}_past_month.png" ${*}
-	rrdtool graph ${RRDARGS} -t "${TITLE} (past year)"  -Y --start=-31536000 --end=-0 "${OUTPNG}_past_year.png"  ${*}
+	rrdtool graph ${RRDARGS} -t "${TITLE} (past hour)"     -v ${YAXIS_NAME} -Y ${TIME_PAST_HOUR}    "${OUTPNG}_past_hour.png"    ${*}
+	rrdtool graph ${RRDARGS} -t "${TITLE} (past 12 hours)" -v ${YAXIS_NAME} -Y ${TIME_PAST_12HOURS} "${OUTPNG}_past_12hours.png" ${*}
+	rrdtool graph ${RRDARGS} -t "${TITLE} (past day)"      -v ${YAXIS_NAME} -Y ${TIME_PAST_DAY}     "${OUTPNG}_past_day.png"     ${*}
+	rrdtool graph ${RRDARGS} -t "${TITLE} (past week)"     -v ${YAXIS_NAME} -Y ${TIME_PAST_WEEK}    "${OUTPNG}_past_week.png"    ${*}
+	rrdtool graph ${RRDARGS} -t "${TITLE} (past month)"    -v ${YAXIS_NAME} -Y ${TIME_PAST_MONTH}   "${OUTPNG}_past_month.png"   ${*}
+	rrdtool graph ${RRDARGS} -t "${TITLE} (past year)"     -v ${YAXIS_NAME} -Y ${TIME_PAST_YEAR}    "${OUTPNG}_past_year.png"    ${*}
 }
 
-rrdtool_graph Flows "${OUTDIR}/flows" \
+rrdtool_graph Flows Amount "${OUTDIR}/flows" \
 	DEF:flows_new=${RRDDIR}/gauge-flow_new_count.rrd:value:AVERAGE \
         DEF:flows_end=${RRDDIR}/gauge-flow_end_count.rrd:value:AVERAGE \
         DEF:flows_idle=${RRDDIR}/gauge-flow_idle_count.rrd:value:AVERAGE \
@@ -68,7 +72,7 @@ rrdtool_graph Flows "${OUTDIR}/flows" \
 	$(rrdtool_graph_print_cur_min_max_avg flows_end) \
         LINE2:flows_idle#CC7016:"Idle" \
 	$(rrdtool_graph_print_cur_min_max_avg flows_idle)
-rrdtool_graph Detections "${OUTDIR}/detections" \
+rrdtool_graph Detections Amount "${OUTDIR}/detections" \
 	DEF:flows_detected=${RRDDIR}/gauge-flow_detected_count.rrd:value:AVERAGE \
 	DEF:flows_guessed=${RRDDIR}/gauge-flow_guessed_count.rrd:value:AVERAGE \
 	DEF:flows_not_detected=${RRDDIR}/gauge-flow_not_detected_count.rrd:value:AVERAGE \
@@ -76,9 +80,9 @@ rrdtool_graph Detections "${OUTDIR}/detections" \
 	DEF:flows_risky=${RRDDIR}/gauge-flow_risky_count.rrd:value:AVERAGE \
 	$(rrdtool_graph_colorize_missing_data flows_detected) \
 	AREA:flows_detected#00bfff::STACK \
+	AREA:flows_detection_update#a1b8c4::STACK \
 	AREA:flows_guessed#ffff4d::STACK \
 	AREA:flows_not_detected#ffa64d::STACK \
-	AREA:flows_detection_update#a1b8c4::STACK \
 	AREA:flows_risky#ff4000::STACK \
 	LINE2:flows_detected#0000ff:"Detected........" \
 	$(rrdtool_graph_print_cur_min_max_avg flows_detected) \
@@ -90,13 +94,15 @@ rrdtool_graph Detections "${OUTDIR}/detections" \
 	$(rrdtool_graph_print_cur_min_max_avg flows_detection_update) \
 	LINE2:flows_risky#b32d00:"Risky..........." \
 	$(rrdtool_graph_print_cur_min_max_avg flows_risky)
-rrdtool_graph "Traffic (IN/OUT)" "${OUTDIR}/traffic" \
-        DEF:total_bytes=${RRDDIR}/gauge-flow_total_bytes.rrd:value:AVERAGE \
-	$(rrdtool_graph_colorize_missing_data total_bytes) \
-	AREA:total_bytes#bea1c4::STACK \
-        LINE2:total_bytes#92629d:"Total-Bytes-Xfer" \
-	$(rrdtool_graph_print_cur_min_max_avg total_bytes)
-rrdtool_graph Layer3-Flows "${OUTDIR}/layer3" \
+rrdtool_graph "Traffic (IN/OUT)" Bytes "${OUTDIR}/traffic" \
+	DEF:total_src_bytes=${RRDDIR}/gauge-flow_src_total_bytes.rrd:value:AVERAGE \
+	DEF:total_dst_bytes=${RRDDIR}/gauge-flow_dst_total_bytes.rrd:value:AVERAGE \
+	$(rrdtool_graph_colorize_missing_data total_src_bytes) \
+	AREA:total_src_bytes#00cc99:"Total-Bytes-Source2Dest":STACK \
+	$(rrdtool_graph_print_cur_min_max_avg total_src_bytes) \
+	STACK:total_dst_bytes#669999:"Total-Bytes-Dest2Source" \
+	$(rrdtool_graph_print_cur_min_max_avg total_dst_bytes)
+rrdtool_graph Layer3-Flows Amount "${OUTDIR}/layer3" \
         DEF:layer3_ip4=${RRDDIR}/gauge-flow_l3_ip4_count.rrd:value:AVERAGE \
 	DEF:layer3_ip6=${RRDDIR}/gauge-flow_l3_ip6_count.rrd:value:AVERAGE \
 	DEF:layer3_other=${RRDDIR}/gauge-flow_l3_other_count.rrd:value:AVERAGE \
@@ -110,7 +116,7 @@ rrdtool_graph Layer3-Flows "${OUTDIR}/layer3" \
 	$(rrdtool_graph_print_cur_min_max_avg layer3_ip6) \
 	LINE2:layer3_other#92629d:"Other" \
 	$(rrdtool_graph_print_cur_min_max_avg layer3_other)
-rrdtool_graph Layer4-Flows "${OUTDIR}/layer4" \
+rrdtool_graph Layer4-Flows Amount "${OUTDIR}/layer4" \
         DEF:layer4_tcp=${RRDDIR}/gauge-flow_l4_tcp_count.rrd:value:AVERAGE \
         DEF:layer4_udp=${RRDDIR}/gauge-flow_l4_udp_count.rrd:value:AVERAGE \
         DEF:layer4_icmp=${RRDDIR}/gauge-flow_l4_icmp_count.rrd:value:AVERAGE \
@@ -128,7 +134,7 @@ rrdtool_graph Layer4-Flows "${OUTDIR}/layer4" \
 	$(rrdtool_graph_print_cur_min_max_avg layer4_icmp) \
 	LINE2:layer4_other#83588d:"Other" \
 	$(rrdtool_graph_print_cur_min_max_avg layer4_other)
-rrdtool_graph Flow-Breeds "${OUTDIR}/breed" \
+rrdtool_graph Flow-Breeds Amount "${OUTDIR}/breed" \
 	DEF:breed_safe=${RRDDIR}/gauge-flow_breed_safe_count.rrd:value:AVERAGE \
 	DEF:breed_acceptable=${RRDDIR}/gauge-flow_breed_acceptable_count.rrd:value:AVERAGE \
 	DEF:breed_fun=${RRDDIR}/gauge-flow_breed_fun_count.rrd:value:AVERAGE \
@@ -162,7 +168,7 @@ rrdtool_graph Flow-Breeds "${OUTDIR}/breed" \
 	$(rrdtool_graph_print_cur_min_max_avg breed_unrated) \
 	LINE2:breed_unknown#ae849a:"Unknown.............." \
 	$(rrdtool_graph_print_cur_min_max_avg breed_unknown)
-rrdtool_graph Flow-Categories "${OUTDIR}/categories" \
+rrdtool_graph Flow-Categories 'Amount(SUM)' "${OUTDIR}/categories" \
 	DEF:cat_ads=${RRDDIR}/gauge-flow_category_advertisment_count.rrd:value:AVERAGE \
 	DEF:cat_chat=${RRDDIR}/gauge-flow_category_chat_count.rrd:value:AVERAGE \
 	DEF:cat_cloud=${RRDDIR}/gauge-flow_category_cloud_count.rrd:value:AVERAGE \
@@ -192,59 +198,59 @@ rrdtool_graph Flow-Categories "${OUTDIR}/categories" \
 	DEF:cat_vpn=${RRDDIR}/gauge-flow_category_vpn_count.rrd:value:AVERAGE \
 	DEF:cat_web=${RRDDIR}/gauge-flow_category_web_count.rrd:value:AVERAGE \
 	$(rrdtool_graph_colorize_missing_data cat_ads) \
-	LINE2:cat_ads#f1c232:"Advertisment..........." \
+	AREA:cat_ads#f1c232:"Advertisment..........." \
 	$(rrdtool_graph_print_cur_min_max_avg cat_ads) \
-	LINE2:cat_chat#6fa8dc:"Chat..................." \
+	STACK:cat_chat#6fa8dc:"Chat..................." \
 	$(rrdtool_graph_print_cur_min_max_avg cat_chat) \
-	LINE2:cat_cloud#2986cc:"Cloud.................." \
+	STACK:cat_cloud#2986cc:"Cloud.................." \
 	$(rrdtool_graph_print_cur_min_max_avg cat_cloud) \
-	LINE2:cat_xfer#16537e:"Data-Transfer.........." \
+	STACK:cat_xfer#16537e:"Data-Transfer.........." \
 	$(rrdtool_graph_print_cur_min_max_avg cat_xfer) \
-	LINE2:cat_db#cc0000:"Database..............." \
+	STACK:cat_db#cc0000:"Database..............." \
 	$(rrdtool_graph_print_cur_min_max_avg cat_db) \
-	LINE2:cat_dl#6a329f:"Download..............." \
+	STACK:cat_dl#6a329f:"Download..............." \
 	$(rrdtool_graph_print_cur_min_max_avg cat_dl) \
-	LINE2:cat_mail#3600cc:"Mail..................." \
+	STACK:cat_mail#3600cc:"Mail..................." \
 	$(rrdtool_graph_print_cur_min_max_avg cat_mail) \
-	LINE2:cat_fs#c90076:"File-Sharing..........." \
+	STACK:cat_fs#c90076:"File-Sharing..........." \
 	$(rrdtool_graph_print_cur_min_max_avg cat_fs) \
-	LINE2:cat_game#00ff26:"Game..................." \
+	STACK:cat_game#00ff26:"Game..................." \
 	$(rrdtool_graph_print_cur_min_max_avg cat_game) \
-	LINE2:cat_mal#f44336:"Malware................" \
+	STACK:cat_mal#f44336:"Malware................" \
 	$(rrdtool_graph_print_cur_min_max_avg cat_mal) \
-	LINE2:cat_med#ff8300:"Media.................." \
+	STACK:cat_med#ff8300:"Media.................." \
 	$(rrdtool_graph_print_cur_min_max_avg cat_med) \
-	LINE2:cat_min#ff0000:"Mining................." \
+	STACK:cat_min#ff0000:"Mining................." \
 	$(rrdtool_graph_print_cur_min_max_avg cat_min) \
-	LINE2:cat_mus#00fff0:"Music.................." \
+	STACK:cat_mus#00fff0:"Music.................." \
 	$(rrdtool_graph_print_cur_min_max_avg cat_mus) \
-	LINE2:cat_net#ddff00:"Network................" \
+	STACK:cat_net#ddff00:"Network................" \
 	$(rrdtool_graph_print_cur_min_max_avg cat_net) \
-	LINE2:cat_oth#744700:"Other.................." \
+	STACK:cat_oth#744700:"Other.................." \
 	$(rrdtool_graph_print_cur_min_max_avg cat_oth) \
-	LINE2:cat_prod#29ff00:"Productivity..........." \
+	STACK:cat_prod#29ff00:"Productivity..........." \
 	$(rrdtool_graph_print_cur_min_max_avg cat_prod) \
-	LINE2:cat_rem#b52c2c:"Remote-Access.........." \
+	STACK:cat_rem#b52c2c:"Remote-Access.........." \
 	$(rrdtool_graph_print_cur_min_max_avg cat_rem) \
-	LINE2:cat_rpc#e15a5a:"Remote-Procedure-Call.." \
+	STACK:cat_rpc#e15a5a:"Remote-Procedure-Call.." \
 	$(rrdtool_graph_print_cur_min_max_avg cat_rpc) \
-	LINE2:cat_shop#0065ff:"Shopping..............." \
+	STACK:cat_shop#0065ff:"Shopping..............." \
 	$(rrdtool_graph_print_cur_min_max_avg cat_shop) \
-	LINE2:cat_soc#8fce00:"Social-Network........." \
+	STACK:cat_soc#8fce00:"Social-Network........." \
 	$(rrdtool_graph_print_cur_min_max_avg cat_soc) \
-	LINE2:cat_soft#007a0d:"Software-Update........" \
+	STACK:cat_soft#007a0d:"Software-Update........" \
 	$(rrdtool_graph_print_cur_min_max_avg cat_soft) \
-	LINE2:cat_str#ff00b8:"Streaming.............." \
+	STACK:cat_str#ff00b8:"Streaming.............." \
 	$(rrdtool_graph_print_cur_min_max_avg cat_str) \
-	LINE2:cat_sys#f4ff00:"System................." \
+	STACK:cat_sys#f4ff00:"System................." \
 	$(rrdtool_graph_print_cur_min_max_avg cat_sys) \
-	LINE2:cat_ukn#999999:"Unknown................" \
+	STACK:cat_ukn#999999:"Unknown................" \
 	$(rrdtool_graph_print_cur_min_max_avg cat_ukn) \
-	LINE2:cat_vid#518820:"Video.................." \
+	STACK:cat_vid#518820:"Video.................." \
 	$(rrdtool_graph_print_cur_min_max_avg cat_vid) \
-	LINE2:cat_voip#ffc700:"Voice-Over-IP.........." \
+	STACK:cat_voip#ffc700:"Voice-Over-IP.........." \
 	$(rrdtool_graph_print_cur_min_max_avg cat_voip) \
-	LINE2:cat_vpn#378035:"Virtual-Private-Network" \
+	STACK:cat_vpn#378035:"Virtual-Private-Network" \
 	$(rrdtool_graph_print_cur_min_max_avg cat_vpn) \
-	LINE2:cat_web#00fffb:"Web...................." \
+	STACK:cat_web#00fffb:"Web...................." \
 	$(rrdtool_graph_print_cur_min_max_avg cat_web)
diff --git a/examples/c-simple/c-simple.c b/examples/c-simple/c-simple.c
index aeb0726c3..9d1ca54b4 100644
--- a/examples/c-simple/c-simple.c
+++ b/examples/c-simple/c-simple.c
@@ -83,8 +83,6 @@ static void nDPIsrvd_verify_flows_cb(struct nDPIsrvd_thread_data const * const t
     {
         fprintf(stderr, "Thread [UNKNOWN], Flow %llu verification failed\n", flow->id_as_ull);
     }
-
-    exit(1);
 }
 
 static void sighandler(int signum)
@@ -108,6 +106,9 @@ static void sighandler(int signum)
         if (verification_failed == 0)
         {
             fprintf(stderr, "%s\n", "Flow verification succeeded.");
+        } else {
+            /* FATAL! */
+            exit(EXIT_FAILURE);
         }
     }
     else if (main_thread_shutdown == 0)
@@ -129,10 +130,19 @@ static enum nDPIsrvd_callback_return simple_json_callback(struct nDPIsrvd_socket
         return CALLBACK_OK;
     }
 
+    struct nDPIsrvd_json_token const * const alias = TOKEN_GET_SZ(sock, "alias");
+    struct nDPIsrvd_json_token const * const source = TOKEN_GET_SZ(sock, "source");
+    if (alias == NULL || source == NULL)
+    {
+        return CALLBACK_ERROR;
+    }
+
     struct nDPIsrvd_json_token const * const flow_event_name = TOKEN_GET_SZ(sock, "flow_event_name");
     if (TOKEN_VALUE_EQUALS_SZ(flow_event_name, "new") != 0)
     {
-        printf("Instance 0x%x, Thread %d, Flow %llu new\n",
+        printf("Instance %.*s/%.*s (HT-Key: 0x%x), Thread %d, Flow %llu new\n",
+               alias->value_length, alias->value,
+               source->value_length, source->value,
                instance->alias_source_key,
                flow->thread_id,
                flow->id_as_ull);
@@ -150,8 +160,19 @@ static void simple_flow_cleanup_callback(struct nDPIsrvd_socket * const sock,
     (void)sock;
     (void)thread_data;
 
+    struct nDPIsrvd_json_token const * const alias = TOKEN_GET_SZ(sock, "alias");
+    struct nDPIsrvd_json_token const * const source = TOKEN_GET_SZ(sock, "source");
+    if (alias == NULL || source == NULL)
+    {
+        /* FATAL! */
+        fprintf(stderr, "BUG: Missing JSON token alias/source.\n");
+        exit(EXIT_FAILURE);
+    }
+
     char const * const reason_str = nDPIsrvd_enum_to_string(reason);
-    printf("Instance 0x%x, Thread %d, Flow %llu cleanup, reason: %s\n",
+    printf("Instance %.*s/%.*s (HT-Key: 0x%x), Thread %d, Flow %llu cleanup, reason: %s\n",
+           alias->value_length, alias->value,
+           source->value_length, source->value,
            instance->alias_source_key,
            flow->thread_id,
            flow->id_as_ull,
@@ -159,7 +180,9 @@ static void simple_flow_cleanup_callback(struct nDPIsrvd_socket * const sock,
 
     if (reason == CLEANUP_REASON_FLOW_TIMEOUT)
     {
+        /* FATAL! */
         fprintf(stderr, "Flow %llu timeouted.\n", flow->id_as_ull);
+        exit(EXIT_FAILURE);
     }
 }
 
diff --git a/examples/py-flow-dashboard/flow-dash.py b/examples/py-flow-dashboard/flow-dash.py
index f2135735d..d396e7e97 100755
--- a/examples/py-flow-dashboard/flow-dash.py
+++ b/examples/py-flow-dashboard/flow-dash.py
@@ -3,9 +3,11 @@
 import multiprocessing
 import os
 import sys
+import time
 
 sys.path.append(os.path.dirname(sys.argv[0]) + '/../../dependencies')
 sys.path.append(os.path.dirname(sys.argv[0]) + '/../share/nDPId')
+sys.path.append(os.path.dirname(sys.argv[0]))
 sys.path.append(sys.base_prefix + '/share/nDPId')
 import nDPIsrvd
 from nDPIsrvd import nDPIsrvdSocket
@@ -165,6 +167,10 @@ def nDPIsrvd_worker_onJsonLineRecvd(json_dict, instance, current_flow, global_us
 
         shared_flow_dict['total-flow-update-events'] += 1
 
+    elif json_dict['flow_event_name'] == 'analyse':
+
+        shared_flow_dict['total-flow-analyse-events'] += 1
+
     elif json_dict['flow_event_name'] == 'end':
 
         shared_flow_dict['total-flow-end-events'] += 1
@@ -231,6 +237,7 @@ def nDPIsrvd_worker(address, shared_flow_dict):
                 sys.stderr.write('Lost connection to {} .. reconnecting\n'
                                  .format(address[0]+':'+str(address[1])
                                          if type(address) is tuple else address))
+                time.sleep(1.0)
     except KeyboardInterrupt:
         pass
 
@@ -248,6 +255,7 @@ if __name__ == '__main__':
     shared_flow_dict['total-events']             = 0
     shared_flow_dict['total-flow-new-events']    = 0
     shared_flow_dict['total-flow-update-events'] = 0
+    shared_flow_dict['total-flow-analyse-events'] = 0
     shared_flow_dict['total-flow-end-events']    = 0
     shared_flow_dict['total-flow-idle-events']   = 0
     shared_flow_dict['total-flow-detected-events'] = 0
diff --git a/examples/py-flow-dashboard/plotly_dash.py b/examples/py-flow-dashboard/plotly_dash.py
index 009d46213..34791d8b5 100644
--- a/examples/py-flow-dashboard/plotly_dash.py
+++ b/examples/py-flow-dashboard/plotly_dash.py
@@ -389,13 +389,14 @@ def tab_flow_update_graph(n, i, traces):
               inputs=[Input('tab-other-default-interval', 'n_intervals')])
 def tab_other_update_components(n):
     return [build_piechart(['Base', 'Daemon', 'Packet',
-                            'Flow New', 'Flow Update', 'Flow End', 'Flow Idle',
+                            'Flow New', 'Flow Update', 'Flow Analyse', 'Flow End', 'Flow Idle',
                             'Flow Detection', 'Flow Detection-Updates', 'Flow Guessed', 'Flow Not-Detected'],
                            [shared_flow_dict['total-base-events'],
                             shared_flow_dict['total-daemon-events'],
                             shared_flow_dict['total-packet-events'],
                             shared_flow_dict['total-flow-new-events'],
                             shared_flow_dict['total-flow-update-events'],
+                            shared_flow_dict['total-flow-analyse-events'],
                             shared_flow_dict['total-flow-end-events'],
                             shared_flow_dict['total-flow-idle-events'],
                             shared_flow_dict['total-flow-detected-events'],
diff --git a/examples/py-flow-info/flow-info.py b/examples/py-flow-info/flow-info.py
index 770058978..4c1c2f909 100755
--- a/examples/py-flow-info/flow-info.py
+++ b/examples/py-flow-info/flow-info.py
@@ -8,6 +8,7 @@ import datetime
 
 sys.path.append(os.path.dirname(sys.argv[0]) + '/../../dependencies')
 sys.path.append(os.path.dirname(sys.argv[0]) + '/../share/nDPId')
+sys.path.append(os.path.dirname(sys.argv[0]))
 sys.path.append(sys.base_prefix + '/share/nDPId')
 import nDPIsrvd
 from nDPIsrvd import nDPIsrvdSocket, TermColor
@@ -62,7 +63,8 @@ class Stats:
         if current_flow is None:
             return
 
-        set_attr_from_dict(current_flow, json_dict, 'flow_tot_l4_payload_len', 0)
+        set_attr_from_dict(current_flow, json_dict, 'flow_src_tot_l4_payload_len', 0)
+        set_attr_from_dict(current_flow, json_dict, 'flow_dst_tot_l4_payload_len', 0)
         if 'ndpi' in json_dict:
             set_attr_from_dict(current_flow, json_dict['ndpi'], 'flow_risk', {})
         else:
@@ -89,7 +91,7 @@ class Stats:
 
     def updateOnCleanup(self, current_flow):
         self.total_flows += 1
-        self.expired_tot_l4_payload_len += current_flow.flow_tot_l4_payload_len
+        self.expired_tot_l4_payload_len += current_flow.flow_src_tot_l4_payload_len + current_flow.flow_dst_tot_l4_payload_len
         self.risky_flows += 1 if len(current_flow.flow_risk) > 0 else 0
         self.midstream_flows += 1 if current_flow.midstream != 0 else 0
         self.guessed_flows += 1 if current_flow.guessed != 0 else 0
@@ -114,7 +116,7 @@ class Stats:
                     flow_count += 1
                     current_flow = instances[alias][source].flows[flow_id]
 
-                    flow_tot_l4_payload_len += current_flow.flow_tot_l4_payload_len
+                    flow_tot_l4_payload_len += current_flow.flow_src_tot_l4_payload_len + current_flow.flow_dst_tot_l4_payload_len
                     risky += 1 if len(current_flow.flow_risk) > 0 else 0
                     midstream += 1 if current_flow.midstream != 0 else 0
                     guessed += 1 if current_flow.guessed != 0 else 0
@@ -182,7 +184,7 @@ def checkEventFilter(json_dict):
                    'guessed': args.guessed, 'detected': args.detected,
                    'detection-update': args.detection_update,
                    'not-detected': args.not_detected,
-                   'update': args.update, 'analysis': args.analysis}
+                   'update': args.update, 'analyse': args.analyse}
 
     if flow_events[json_dict['flow_event_name']] is True:
         return True
@@ -249,7 +251,7 @@ def onJsonLineRecvd(json_dict, instance, current_flow, global_user_data):
         basic_daemon_event_prefix += ' ' * 11
         if 'flow_first_seen' in json_dict:
             first_seen = '[' + prettifyTimediff(nDPIsrvd.toSeconds(json_dict['flow_first_seen']),
-                                                nDPIsrvd.toSeconds(json_dict['thread_ts_usec']) + ']'
+                                                nDPIsrvd.toSeconds(json_dict['thread_ts_usec']) + ']')
 
     last_seen = ''
     if args.print_last_seen is True:
@@ -257,7 +259,7 @@ def onJsonLineRecvd(json_dict, instance, current_flow, global_user_data):
         if current_flow is not None:
             flow_last_seen = nDPIsrvd.FlowManager.getLastPacketTime(instance, current_flow.flow_id, json_dict)
             last_seen = '[' + prettifyTimediff(nDPIsrvd.toSeconds(flow_last_seen),
-                                               nDPIsrvd.toSeconds(json_dict['thread_ts_usec']) + ']'
+                                               nDPIsrvd.toSeconds(json_dict['thread_ts_usec']) + ']')
 
     if 'daemon_event_id' in json_dict:
         if json_dict['daemon_event_name'] == 'status':
@@ -346,8 +348,8 @@ def onJsonLineRecvd(json_dict, instance, current_flow, global_user_data):
     elif json_dict['flow_event_name'] == 'not-detected':
         flow_event_name += '{}{:>16}{}'.format(TermColor.WARNING + TermColor.BOLD + TermColor.BLINK,
                                                json_dict['flow_event_name'], TermColor.END)
-    elif json_dict['flow_event_name'] == 'analysis':
-        flow_event_name += '{}{:>16}{}'.format(TermColor.WARNING + TermColor.BLINK,
+    elif json_dict['flow_event_name'] == 'analyse':
+        flow_event_name += '{}{:>16}{}'.format(TermColor.WARNING,
                                                json_dict['flow_event_name'], TermColor.END)
     else:
         if json_dict['flow_event_name'] == 'new':
@@ -418,7 +420,7 @@ if __name__ == '__main__':
     argparser.add_argument('--end',        action='store_true', default=False, help='Print only end flow events.')
     argparser.add_argument('--idle',       action='store_true', default=False, help='Print only idle flow events.')
     argparser.add_argument('--update',     action='store_true', default=False, help='Print only update flow events.')
-    argparser.add_argument('--analysis',   action='store_true', default=False, help='Print only analysis flow events.')
+    argparser.add_argument('--analyse',    action='store_true', default=False, help='Print only analyse flow events.')
     argparser.add_argument('--detection',  action='store_true', default=False, help='Print only detected/detection-update flow events.')
     argparser.add_argument('--ipwhois',    action='store_true', default=False, help='Use Python-IPWhois to print additional location information.')
     args = argparser.parse_args()
diff --git a/examples/py-flow-muliprocess/py-flow-multiprocess.py b/examples/py-flow-muliprocess/py-flow-multiprocess.py
index 9014c5f59..3313b156b 100755
--- a/examples/py-flow-muliprocess/py-flow-multiprocess.py
+++ b/examples/py-flow-muliprocess/py-flow-multiprocess.py
@@ -6,7 +6,8 @@ import sys
 
 sys.path.append(os.path.dirname(sys.argv[0]) + '/../../dependencies')
 sys.path.append(os.path.dirname(sys.argv[0]) + '/../share/nDPId')
-sys.path.append(os.path.dirname(sys.argv[0]) + '/../usr/share/nDPId')
+sys.path.append(os.path.dirname(sys.argv[0]))
+sys.path.append(sys.base_prefix + '/share/nDPId')
 import nDPIsrvd
 from nDPIsrvd import nDPIsrvdSocket
 
diff --git a/examples/py-ja3-checker/py-ja3-checker.py b/examples/py-ja3-checker/py-ja3-checker.py
index d3db3b56e..cf455fbf1 100755
--- a/examples/py-ja3-checker/py-ja3-checker.py
+++ b/examples/py-ja3-checker/py-ja3-checker.py
@@ -10,7 +10,8 @@ import time
 
 sys.path.append(os.path.dirname(sys.argv[0]) + '/../../dependencies')
 sys.path.append(os.path.dirname(sys.argv[0]) + '/../share/nDPId')
-sys.path.append(os.path.dirname(sys.argv[0]) + '/../usr/share/nDPId')
+sys.path.append(os.path.dirname(sys.argv[0]))
+sys.path.append(sys.base_prefix + '/share/nDPId')
 import nDPIsrvd
 from nDPIsrvd import nDPIsrvdSocket
 
diff --git a/examples/py-json-stdout/json-stdout.py b/examples/py-json-stdout/json-stdout.py
index bd27d7d80..f1aa51b5b 100755
--- a/examples/py-json-stdout/json-stdout.py
+++ b/examples/py-json-stdout/json-stdout.py
@@ -5,7 +5,8 @@ import sys
 
 sys.path.append(os.path.dirname(sys.argv[0]) + '/../../dependencies')
 sys.path.append(os.path.dirname(sys.argv[0]) + '/../share/nDPId')
-sys.path.append(os.path.dirname(sys.argv[0]) + '/../usr/share/nDPId')
+sys.path.append(os.path.dirname(sys.argv[0]))
+sys.path.append(sys.base_prefix + '/share/nDPId')
 import nDPIsrvd
 from nDPIsrvd import nDPIsrvdSocket, TermColor
 
diff --git a/examples/py-schema-validation/py-schema-validation.py b/examples/py-schema-validation/py-schema-validation.py
index 4a1856895..fea2df4d1 100755
--- a/examples/py-schema-validation/py-schema-validation.py
+++ b/examples/py-schema-validation/py-schema-validation.py
@@ -5,7 +5,8 @@ import sys
 
 sys.path.append(os.path.dirname(sys.argv[0]) + '/../../dependencies')
 sys.path.append(os.path.dirname(sys.argv[0]) + '/../share/nDPId')
-sys.path.append(os.path.dirname(sys.argv[0]) + '/../usr/share/nDPId')
+sys.path.append(os.path.dirname(sys.argv[0]))
+sys.path.append(sys.base_prefix + '/share/nDPId')
 import nDPIsrvd
 from nDPIsrvd import nDPIsrvdSocket, TermColor
 
diff --git a/examples/py-semantic-validation/py-semantic-validation.py b/examples/py-semantic-validation/py-semantic-validation.py
index 01e4faf11..450500dc5 100755
--- a/examples/py-semantic-validation/py-semantic-validation.py
+++ b/examples/py-semantic-validation/py-semantic-validation.py
@@ -5,11 +5,18 @@ import sys
 
 sys.path.append(os.path.dirname(sys.argv[0]) + '/../../dependencies')
 sys.path.append(os.path.dirname(sys.argv[0]) + '/../share/nDPId')
-sys.path.append(os.path.dirname(sys.argv[0]) + '/../usr/share/nDPId')
+sys.path.append(os.path.dirname(sys.argv[0]))
+sys.path.append(sys.base_prefix + '/share/nDPId')
 import nDPIsrvd
 from nDPIsrvd import nDPIsrvdSocket, TermColor
 
 class Stats:
+    KEYS = [ ['init','reconnect','shutdown','status' ], \
+            [ 'new','end','idle','update', ],
+            [ 'analyse' ], \
+            [ 'guessed','detected','detection-update','not-detected' ], \
+            [ 'packet', 'packet-flow'] ]
+    ALL_KEYS = KEYS[0] + KEYS[1] + KEYS[2] + KEYS[3] + KEYS[4]
 
     def __init__(self, nDPIsrvd_sock):
         self.nsock = nDPIsrvd_sock
@@ -20,11 +27,7 @@ class Stats:
         self.print_nmb_every = self.print_dot_every * 5
 
     def resetEventCounter(self):
-        keys = ['init','reconnect','shutdown','status', \
-                'new','end','idle','update','analyse', \
-                'guessed','detected','detection-update','not-detected', \
-                'packet', 'packet-flow']
-        for k in keys:
+        for k in Stats.ALL_KEYS:
             self.event_counter[k] = 0
 
     def incrementEventCounter(self, json_dict):
@@ -52,13 +55,9 @@ class Stats:
         return True
 
     def getEventCounterStr(self):
-        keys = [ [ 'init','reconnect','shutdown','status' ], \
-                 [ 'new','end','idle','update' ], \
-                 [ 'guessed','detected','detection-update','not-detected' ], \
-                 [ 'packet', 'packet-flow' ] ]
         retval = str()
         retval += '-' * 98 + '--\n'
-        for klist in keys:
+        for klist in Stats.KEYS:
             for k in klist:
                 retval += '| {:<16}: {:<4} '.format(k, self.event_counter[k])
             retval += '\n--' + '-' * 98 + '\n'