nDPId-test: added JSON distribution + JSON parsing (Multithreaded design re-using most of nDPId/nDPIsrvd core)

 * improved Makefile.old install targets
 * splitted nDPIsrvd_parse into nDPIsrvd_parse_line and nDPIsrvd_parse_all for the sake of readability
 * minor Python script improvments (check for nDPIsrvd.py on multiple locations, may be superseeded by setuptools in the future)
 * some paths needs to be absolute (chdir() during daemonize) and therefor additional checks introduced
 * test run script checks and fails if certain files are are missing (PCAP file <=> result output file)
 * removed not very useful "internal format error" JSON serialization if a BUG for same exists
 * fixed invalid l4 type statistics counters for nDPIsrvd-collectd

Signed-off-by: Toni Uhlig <matzeton@googlemail.com>

7 files changed, 63 insertions, 23 deletions

diff --git a/examples/c-captured/c-captured.c b/examples/c-captured/c-captured.c
index b5f7646a3..a427a38fc 100644
--- a/examples/c-captured/c-captured.c
+++ b/examples/c-captured/c-captured.c
@@ -560,6 +560,14 @@ static int parse_options(int argc, char ** argv)
     }
 
     errno = 0;
+    if (datadir[0] != '/')
+    {
+        fprintf(stderr,
+                "%s: PCAP capture directory must be absolut i.e. starting with `/', path given: `%s'\n",
+                argv[0],
+                datadir);
+        return 1;
+    }
     if (mkdir(datadir, S_IRWXU) != 0 && errno != EEXIST)
     {
         fprintf(stderr, "%s: Could not create directory %s: %s\n", argv[0], datadir, strerror(errno));
@@ -581,8 +589,8 @@ static int mainloop(void)
             return 1;
         }
 
-        enum nDPIsrvd_parse_return parse_ret = nDPIsrvd_parse(sock);
-        if (parse_ret != PARSE_OK)
+        enum nDPIsrvd_parse_return parse_ret = nDPIsrvd_parse_all(sock);
+        if (parse_ret != PARSE_NEED_MORE_DATA)
         {
             syslog(LOG_DAEMON | LOG_ERR, "nDPIsrvd parse failed with: %s", nDPIsrvd_enum_to_string(parse_ret));
             return 1;
@@ -645,6 +653,7 @@ int main(int argc, char ** argv)
     int retval = mainloop();
 
     nDPIsrvd_free(&sock);
+    daemonize_shutdown(pidfile);
     closelog();
 
     return retval;
diff --git a/examples/c-collectd/c-collectd.c b/examples/c-collectd/c-collectd.c
index 291676817..42e8b6020 100644
--- a/examples/c-collectd/c-collectd.c
+++ b/examples/c-collectd/c-collectd.c
@@ -89,6 +89,7 @@ static struct
     uint64_t flow_l3_other_count;
     uint64_t flow_l4_tcp_count;
     uint64_t flow_l4_udp_count;
+    uint64_t flow_l4_icmp_count;
     uint64_t flow_l4_other_count;
 } collectd_statistics = {};
 
@@ -313,13 +314,15 @@ static void print_collectd_exec_output(void)
 
     printf(COLLECTD_PUTVAL_N_FORMAT(flow_l3_ip4_count) COLLECTD_PUTVAL_N_FORMAT(flow_l3_ip6_count)
                COLLECTD_PUTVAL_N_FORMAT(flow_l3_other_count) COLLECTD_PUTVAL_N_FORMAT(flow_l4_tcp_count)
-                   COLLECTD_PUTVAL_N_FORMAT(flow_l4_udp_count) COLLECTD_PUTVAL_N_FORMAT(flow_l4_other_count),
+                   COLLECTD_PUTVAL_N_FORMAT(flow_l4_udp_count) COLLECTD_PUTVAL_N_FORMAT(flow_l4_icmp_count)
+                       COLLECTD_PUTVAL_N_FORMAT(flow_l4_other_count),
 
            COLLECTD_PUTVAL_N(flow_l3_ip4_count),
            COLLECTD_PUTVAL_N(flow_l3_ip6_count),
            COLLECTD_PUTVAL_N(flow_l3_other_count),
            COLLECTD_PUTVAL_N(flow_l4_tcp_count),
            COLLECTD_PUTVAL_N(flow_l4_udp_count),
+           COLLECTD_PUTVAL_N(flow_l4_icmp_count),
            COLLECTD_PUTVAL_N(flow_l4_other_count));
 
     memset(&collectd_statistics, 0, sizeof(collectd_statistics));
@@ -370,8 +373,8 @@ static int mainloop(int epollfd)
                     return 1;
                 }
 
-                enum nDPIsrvd_parse_return parse_ret = nDPIsrvd_parse(sock);
-                if (parse_ret != PARSE_OK)
+                enum nDPIsrvd_parse_return parse_ret = nDPIsrvd_parse_all(sock);
+                if (parse_ret != PARSE_NEED_MORE_DATA)
                 {
                     LOG(LOG_DAEMON | LOG_ERR, "nDPIsrvd parse failed with: %s", nDPIsrvd_enum_to_string(parse_ret));
                     return 1;
@@ -424,14 +427,18 @@ static enum nDPIsrvd_callback_return captured_json_callback(struct nDPIsrvd_sock
         }
 
         struct nDPIsrvd_json_token const * const l4_proto = TOKEN_GET_SZ(sock, "l4_proto");
-        if (TOKEN_VALUE_EQUALS_SZ(l3_proto, "tcp") != 0)
+        if (TOKEN_VALUE_EQUALS_SZ(l4_proto, "tcp") != 0)
         {
             collectd_statistics.flow_l4_tcp_count++;
         }
-        else if (TOKEN_VALUE_EQUALS_SZ(l3_proto, "tcp") != 0)
+        else if (TOKEN_VALUE_EQUALS_SZ(l4_proto, "udp") != 0)
         {
             collectd_statistics.flow_l4_udp_count++;
         }
+        else if (TOKEN_VALUE_EQUALS_SZ(l4_proto, "icmp") != 0)
+        {
+            collectd_statistics.flow_l4_icmp_count++;
+        }
         else if (l4_proto != NULL)
         {
             collectd_statistics.flow_l4_other_count++;
diff --git a/examples/py-flow-info/flow-info.py b/examples/py-flow-info/flow-info.py
index f9b71df91..3b11a03b7 100755
--- a/examples/py-flow-info/flow-info.py
+++ b/examples/py-flow-info/flow-info.py
@@ -3,9 +3,14 @@
 import os
 import sys
 
-sys.path.append(os.path.dirname(sys.argv[0]) + '/../../dependencies')
-import nDPIsrvd
-from nDPIsrvd import nDPIsrvdSocket, TermColor
+sys.path.append(os.path.dirname(sys.argv[0]) + '/../usr/share/nDPId')
+try:
+    import nDPIsrvd
+    from nDPIsrvd import nDPIsrvdSocket, TermColor
+except ModuleNotFoundError:
+    sys.path.append(os.path.dirname(sys.argv[0]) + '/../../dependencies')
+    import nDPIsrvd
+    from nDPIsrvd import nDPIsrvdSocket, TermColor
 
 global args
 
diff --git a/examples/py-flow-undetected-to-pcap/flow-undetected-to-pcap.py b/examples/py-flow-undetected-to-pcap/flow-undetected-to-pcap.py
index 961adc3cf..c25a7601f 100755
--- a/examples/py-flow-undetected-to-pcap/flow-undetected-to-pcap.py
+++ b/examples/py-flow-undetected-to-pcap/flow-undetected-to-pcap.py
@@ -3,9 +3,14 @@
 import os
 import sys
 
-sys.path.append(os.path.dirname(sys.argv[0]) + '/../../dependencies')
-import nDPIsrvd
-from nDPIsrvd import TermColor, nDPIsrvdSocket, PcapPacket
+sys.path.append(os.path.dirname(sys.argv[0]) + '/../usr/share/nDPId')
+try:
+    import nDPIsrvd
+    from nDPIsrvd import nDPIsrvdSocket, TermColor
+except ModuleNotFoundError:
+    sys.path.append(os.path.dirname(sys.argv[0]) + '/../../dependencies')
+    import nDPIsrvd
+    from nDPIsrvd import nDPIsrvdSocket, TermColor
 
 def onJsonLineRecvd(json_dict, current_flow, global_user_data):
     if current_flow is None:
diff --git a/examples/py-json-stdout/json-stdout.py b/examples/py-json-stdout/json-stdout.py
index 9f58d161e..a14447745 100755
--- a/examples/py-json-stdout/json-stdout.py
+++ b/examples/py-json-stdout/json-stdout.py
@@ -3,10 +3,14 @@
 import os
 import sys
 
-sys.path.append(os.path.dirname(sys.argv[0]) + '/../../dependencies')
-import nDPIsrvd
-from nDPIsrvd import nDPIsrvdSocket, TermColor
-
+sys.path.append(os.path.dirname(sys.argv[0]) + '/../usr/share/nDPId')
+try:
+    import nDPIsrvd
+    from nDPIsrvd import nDPIsrvdSocket, TermColor
+except ModuleNotFoundError:
+    sys.path.append(os.path.dirname(sys.argv[0]) + '/../../dependencies')
+    import nDPIsrvd
+    from nDPIsrvd import nDPIsrvdSocket, TermColor
 
 def onJsonLineRecvd(json_dict, current_flow, global_user_data):
     print(json_dict)
diff --git a/examples/py-risky-flow-to-pcap/risky-flow-to-pcap.py b/examples/py-risky-flow-to-pcap/risky-flow-to-pcap.py
index 33c0be810..11165ed35 100755
--- a/examples/py-risky-flow-to-pcap/risky-flow-to-pcap.py
+++ b/examples/py-risky-flow-to-pcap/risky-flow-to-pcap.py
@@ -4,9 +4,14 @@ import base64
 import os
 import sys
 
-sys.path.append(os.path.dirname(sys.argv[0]) + '/../../dependencies')
-import nDPIsrvd
-from nDPIsrvd import TermColor, nDPIsrvdSocket, PcapPacket
+sys.path.append(os.path.dirname(sys.argv[0]) + '/../usr/share/nDPId')
+try:
+    import nDPIsrvd
+    from nDPIsrvd import nDPIsrvdSocket, TermColor
+except ModuleNotFoundError:
+    sys.path.append(os.path.dirname(sys.argv[0]) + '/../../dependencies')
+    import nDPIsrvd
+    from nDPIsrvd import nDPIsrvdSocket, TermColor
 
 def onJsonLineRecvd(json_dict, current_flow, global_user_data):
     if current_flow is None:
diff --git a/examples/py-schema-validation/py-schema-validation.py b/examples/py-schema-validation/py-schema-validation.py
index ca269e0c3..583612516 100755
--- a/examples/py-schema-validation/py-schema-validation.py
+++ b/examples/py-schema-validation/py-schema-validation.py
@@ -3,9 +3,14 @@
 import os
 import sys
 
-sys.path.append(os.path.dirname(sys.argv[0]) + '/../../dependencies')
-import nDPIsrvd
-from nDPIsrvd import nDPIsrvdSocket, TermColor
+sys.path.append(os.path.dirname(sys.argv[0]) + '/../usr/share/nDPId')
+try:
+    import nDPIsrvd
+    from nDPIsrvd import nDPIsrvdSocket, TermColor
+except ModuleNotFoundError:
+    sys.path.append(os.path.dirname(sys.argv[0]) + '/../../dependencies')
+    import nDPIsrvd
+    from nDPIsrvd import nDPIsrvdSocket, TermColor
 
 class Stats:
     lines_processed = 0