Improved flown analyse event:

 * store packet directions
 * merged direction based IATs
 * merged direction based PKTLENs

Signed-off-by: Toni Uhlig <matzeton@googlemail.com>

5 files changed, 151 insertions, 35 deletions

diff --git a/examples/c-captured/c-captured.c b/examples/c-captured/c-captured.c
index d7e7600c3..a99105253 100644
--- a/examples/c-captured/c-captured.c
+++ b/examples/c-captured/c-captured.c
@@ -72,6 +72,16 @@ static uint8_t process_midstream = 0;
 static uint8_t ignore_empty_flows = 0;
 
 #ifdef ENABLE_MEMORY_PROFILING
+void nDPIsrvd_memprof_log_alloc(size_t alloc_size)
+{
+    (void)alloc_size;
+}
+
+void nDPIsrvd_memprof_log_free(size_t free_size)
+{
+    (void)free_size;
+}
+
 void nDPIsrvd_memprof_log(char const * const format, ...)
 {
     va_list ap;
diff --git a/examples/c-collectd/c-collectd.c b/examples/c-collectd/c-collectd.c
index bd880d45c..811eeee08 100644
--- a/examples/c-collectd/c-collectd.c
+++ b/examples/c-collectd/c-collectd.c
@@ -104,6 +104,16 @@ static struct
 } collectd_statistics = {};
 
 #ifdef ENABLE_MEMORY_PROFILING
+void nDPIsrvd_memprof_log_alloc(size_t alloc_size)
+{
+    (void)alloc_size;
+}
+
+void nDPIsrvd_memprof_log_free(size_t free_size)
+{
+    (void)free_size;
+}
+
 void nDPIsrvd_memprof_log(char const * const format, ...)
 {
     va_list ap;
diff --git a/examples/c-simple/c-simple.c b/examples/c-simple/c-simple.c
index 9d1ca54b4..abf2e3feb 100644
--- a/examples/c-simple/c-simple.c
+++ b/examples/c-simple/c-simple.c
@@ -9,6 +9,16 @@ static int main_thread_shutdown = 0;
 static struct nDPIsrvd_socket * sock = NULL;
 
 #ifdef ENABLE_MEMORY_PROFILING
+void nDPIsrvd_memprof_log_alloc(size_t alloc_size)
+{
+    (void)alloc_size;
+}
+
+void nDPIsrvd_memprof_log_free(size_t free_size)
+{
+    (void)free_size;
+}
+
 void nDPIsrvd_memprof_log(char const * const format, ...)
 {
     va_list ap;
@@ -106,7 +116,9 @@ static void sighandler(int signum)
         if (verification_failed == 0)
         {
             fprintf(stderr, "%s\n", "Flow verification succeeded.");
-        } else {
+        }
+        else
+        {
             /* FATAL! */
             exit(EXIT_FAILURE);
         }
@@ -141,8 +153,10 @@ static enum nDPIsrvd_callback_return simple_json_callback(struct nDPIsrvd_socket
     if (TOKEN_VALUE_EQUALS_SZ(flow_event_name, "new") != 0)
     {
         printf("Instance %.*s/%.*s (HT-Key: 0x%x), Thread %d, Flow %llu new\n",
-               alias->value_length, alias->value,
-               source->value_length, source->value,
+               alias->value_length,
+               alias->value,
+               source->value_length,
+               source->value,
                instance->alias_source_key,
                flow->thread_id,
                flow->id_as_ull);
@@ -171,8 +185,10 @@ static void simple_flow_cleanup_callback(struct nDPIsrvd_socket * const sock,
 
     char const * const reason_str = nDPIsrvd_enum_to_string(reason);
     printf("Instance %.*s/%.*s (HT-Key: 0x%x), Thread %d, Flow %llu cleanup, reason: %s\n",
-           alias->value_length, alias->value,
-           source->value_length, source->value,
+           alias->value_length,
+           alias->value,
+           source->value_length,
+           source->value,
            instance->alias_source_key,
            flow->thread_id,
            flow->id_as_ull,
diff --git a/examples/py-flow-info/flow-info.py b/examples/py-flow-info/flow-info.py
index 80f7d2692..686ba2436 100755
--- a/examples/py-flow-info/flow-info.py
+++ b/examples/py-flow-info/flow-info.py
@@ -363,43 +363,34 @@ def onJsonLineRecvd(json_dict, instance, current_flow, global_user_data):
         flow_event_name += '{}{:>16}{}'.format(TermColor.WARNING,
                                                json_dict['flow_event_name'], TermColor.END)
         if args.print_analyse_results is True:
-            next_lines = ['[min|max|avg|stddev]']
-            next_lines += ['[IAT(flow)...: {:>8.3f}|{:>8.3f}|{:>8.3f}|{:>8.3f}]'.format(
-                            nDPIsrvd.toSeconds(json_dict['data_analysis']['iat']['flow_min']),
-                            nDPIsrvd.toSeconds(json_dict['data_analysis']['iat']['flow_max']),
-                            nDPIsrvd.toSeconds(json_dict['data_analysis']['iat']['flow_avg']),
-                            nDPIsrvd.toSeconds(json_dict['data_analysis']['iat']['flow_stddev'])
+            next_lines = ['[min|max|avg|stddev|variance|entropy]']
+            next_lines += ['[IAT.........: {:>9.3f}|{:>9.3f}|{:>9.3f}|{:>9.3f}|{:>9.3f}|{:>9.3f}]'.format(
+                            nDPIsrvd.toSeconds(json_dict['data_analysis']['iat']['min']),
+                            nDPIsrvd.toSeconds(json_dict['data_analysis']['iat']['max']),
+                            nDPIsrvd.toSeconds(json_dict['data_analysis']['iat']['avg']),
+                            nDPIsrvd.toSeconds(json_dict['data_analysis']['iat']['stddev']),
+                            nDPIsrvd.toSeconds(json_dict['data_analysis']['iat']['var']),
+                            nDPIsrvd.toSeconds(json_dict['data_analysis']['iat']['ent'])
                           )]
             next_lines += ['']
-            next_lines[-1] += '[IAT(c->s)...: {:>8.3f}|{:>8.3f}|{:>8.3f}|{:>8.3f}]'.format(
-                                    nDPIsrvd.toSeconds(json_dict['data_analysis']['iat']['c_to_s_min']),
-                                    nDPIsrvd.toSeconds(json_dict['data_analysis']['iat']['c_to_s_max']),
-                                    nDPIsrvd.toSeconds(json_dict['data_analysis']['iat']['c_to_s_avg']),
-                                    nDPIsrvd.toSeconds(json_dict['data_analysis']['iat']['c_to_s_stddev'])
-                              )
-            next_lines[-1] += '[IAT(s->c)...: {:>8.3f}|{:>8.3f}|{:>8.3f}|{:>8.3f}]'.format(
-                                    nDPIsrvd.toSeconds(json_dict['data_analysis']['iat']['s_to_c_min']),
-                                    nDPIsrvd.toSeconds(json_dict['data_analysis']['iat']['s_to_c_max']),
-                                    nDPIsrvd.toSeconds(json_dict['data_analysis']['iat']['s_to_c_avg']),
-                                    nDPIsrvd.toSeconds(json_dict['data_analysis']['iat']['s_to_c_stddev'])
-                              )
-            next_lines += ['']
-            next_lines[-1] += '[PKTLEN(c->s): {:>8.3f}|{:>8.3f}|{:>8.3f}|{:>8.3f}]'.format(
-                                  json_dict['data_analysis']['pktlen']['c_to_s_min'],
-                                  json_dict['data_analysis']['pktlen']['c_to_s_max'],
-                                  json_dict['data_analysis']['pktlen']['c_to_s_avg'],
-                                  json_dict['data_analysis']['pktlen']['c_to_s_stddev']
-                              )
-            next_lines[-1] += '[PKTLEN(s->c): {:>8.3f}|{:>8.3f}|{:>8.3f}|{:>8.3f}]'.format(
-                                  json_dict['data_analysis']['pktlen']['s_to_c_min'],
-                                  json_dict['data_analysis']['pktlen']['s_to_c_max'],
-                                  json_dict['data_analysis']['pktlen']['s_to_c_avg'],
-                                  json_dict['data_analysis']['pktlen']['s_to_c_stddev']
+            next_lines[-1] += '[PKTLEN......: {:>9.3f}|{:>9.3f}|{:>9.3f}|{:>9.3f}|{:>9.3f}|{:>9.3f}]'.format(
+                                  json_dict['data_analysis']['pktlen']['min'],
+                                  json_dict['data_analysis']['pktlen']['max'],
+                                  json_dict['data_analysis']['pktlen']['avg'],
+                                  json_dict['data_analysis']['pktlen']['stddev'],
+                                  json_dict['data_analysis']['pktlen']['var'],
+                                  json_dict['data_analysis']['pktlen']['ent']
                               )
             next_lines += ['']
             next_lines[-1] += '[BINS(c->s)..: {}]'.format(','.join([str(n) for n in json_dict['data_analysis']['bins']['c_to_s']]))
             next_lines += ['']
             next_lines[-1] += '[BINS(s->c)..: {}]'.format(','.join([str(n) for n in json_dict['data_analysis']['bins']['s_to_c']]))
+            next_lines += ['']
+            next_lines[-1] += '[DIRECTIONS..: {}]'.format(','.join([str(n) for n in json_dict['data_analysis']['directions']]))
+            next_lines += ['']
+            next_lines[-1] += '[IATS........: {}]'.format(','.join([str(n) for n in json_dict['data_analysis']['iat']['data']]))
+            next_lines += ['']
+            next_lines[-1] += '[PKTLENS.....: {}]'.format(','.join([str(n) for n in json_dict['data_analysis']['pktlen']['data']]))
     else:
         if json_dict['flow_event_name'] == 'new':
             line_suffix = ''
diff --git a/examples/py-machine-learning/sklearn-ml.py b/examples/py-machine-learning/sklearn-ml.py
new file mode 100755
index 000000000..301f4e907
--- /dev/null
+++ b/examples/py-machine-learning/sklearn-ml.py
@@ -0,0 +1,89 @@
+#!/usr/bin/env python3
+
+# pip3 install -U scikit-learn scipy matplotlib
+
+import os
+import sklearn
+import sklearn.ensemble
+import sys
+
+sys.path.append(os.path.dirname(sys.argv[0]) + '/../../dependencies')
+sys.path.append(os.path.dirname(sys.argv[0]) + '/../share/nDPId')
+sys.path.append(os.path.dirname(sys.argv[0]))
+sys.path.append(sys.base_prefix + '/share/nDPId')
+import nDPIsrvd
+from nDPIsrvd import nDPIsrvdSocket, TermColor
+
+class RFC(sklearn.ensemble.RandomForestClassifier):
+    def __init__(self, max_samples):
+        self.max_samples = max_samples
+        self.samples_x = []
+        self.samples_y = []
+        super().__init__(verbose=1, n_estimators=1000, max_samples=max_samples)
+
+    def addSample(self, x, y):
+        self.samples_x += x
+        self.samples_y += y
+
+    def fit(self):
+        if len(self.samples_x) != self.max_samples or \
+           len(self.samples_y) != self.max_samples:
+            return False
+
+        super().fit(self.samples_x, self.samples_y)
+        self.samples_x = []
+        self.samples_y = []
+        return True
+
+def onJsonLineRecvd(json_dict, instance, current_flow, global_user_data):
+    if 'flow_event_name' not in json_dict:
+        return True
+    if json_dict['flow_event_name'] != 'analyse':
+        return True
+
+    if 'ndpi' not in json_dict:
+        return True
+    if 'proto' not in json_dict['ndpi']:
+        return True
+
+    #print(json_dict)
+
+    features = [[]]
+    features[0] += json_dict['data_analysis']['bins']['c_to_s']
+    features[0] += json_dict['data_analysis']['bins']['s_to_c']
+    #print(features)
+
+    out = ''
+    rfc = global_user_data
+    try:
+        out += '[Predict: {}]'.format(rfc.predict(features)[0])
+    except sklearn.exceptions.NotFittedError:
+        pass
+
+    # TLS.DoH_DoT
+    if json_dict['ndpi']['proto'].startswith('TLS.') is not True and \
+       json_dict['ndpi']['proto'] != 'TLS':
+        rfc.addSample(features, [0])
+    else:
+        rfc.addSample(features, [1])
+
+    if rfc.fit() is True:
+        out += '*** FIT *** '
+    out += '[{}]'.format(json_dict['ndpi']['proto'])
+    print(out)
+
+    return True
+
+if __name__ == '__main__':
+    argparser = nDPIsrvd.defaultArgumentParser()
+    args = argparser.parse_args()
+    address = nDPIsrvd.validateAddress(args)
+
+    sys.stderr.write('Recv buffer size: {}\n'.format(nDPIsrvd.NETWORK_BUFFER_MAX_SIZE))
+    sys.stderr.write('Connecting to {} ..\n'.format(address[0]+':'+str(address[1]) if type(address) is tuple else address))
+
+    rfc = RFC(10)
+
+    nsock = nDPIsrvdSocket()
+    nsock.connect(address)
+    nsock.loop(onJsonLineRecvd, None, rfc)