New flow event: 'analysis'.

 * The goal was to provide a separate event for extracted feature that are not required
   and only useful for a few (e.g. someone who wants do ML).
 * Increased network buffer size to 32kB (8192 * 4).
 * Switched timestamp precision from ms to us for *ALL* timestamps.

Signed-off-by: Toni Uhlig <matzeton@googlemail.com>

4 files changed, 124 insertions, 19 deletions

diff --git a/schema/daemon_event_schema.json b/schema/daemon_event_schema.json
index 1eaa7fd03..54f93f98b 100644
--- a/schema/daemon_event_schema.json
+++ b/schema/daemon_event_schema.json
@@ -7,7 +7,7 @@
         "packet_id",
         "daemon_event_id",
         "daemon_event_name",
-        "global_ts_msec"
+        "global_ts_usec"
     ],
     "if": {
         "properties": { "daemon_event_name": { "enum": [ "init", "reconnect" ] } }
@@ -151,9 +151,15 @@
             "type": "number",
             "minimum": 1
         },
-        "global_ts_msec": {
+        "global_ts_usec": {
             "type": "number",
-            "minimum": 0
+            "if": {
+                "properties": { "daemon_event_name": { "enum": [ "init" ] } }
+            },
+            "then" : true,
+            "else" : {
+                "minimum": 1000000
+            }
         }
     },
     "additionalProperties": false
diff --git a/schema/error_event_schema.json b/schema/error_event_schema.json
index a9080d13f..ebeb695fa 100644
--- a/schema/error_event_schema.json
+++ b/schema/error_event_schema.json
@@ -7,7 +7,7 @@
         "error_event_id",
         "error_event_name",
         "datalink",
-        "global_ts_msec"
+        "global_ts_usec"
     ],
 
     "if": {
@@ -154,7 +154,7 @@
         "max_idle": {
             "type": "number"
         },
-        "global_ts_msec": {
+        "global_ts_usec": {
             "type": "number",
             "minimum": 0
         }
diff --git a/schema/flow_event_schema.json b/schema/flow_event_schema.json
index 649e21891..94f2a00df 100644
--- a/schema/flow_event_schema.json
+++ b/schema/flow_event_schema.json
@@ -12,7 +12,8 @@
         "flow_src_packets_processed",
         "flow_dst_packets_processed",
         "flow_first_seen",
-        "flow_last_seen",
+        "flow_src_last_pkt_time",
+        "flow_dst_last_pkt_time",
         "flow_idle_time",
         "flow_src_min_l4_payload_len",
         "flow_dst_min_l4_payload_len",
@@ -23,7 +24,7 @@
         "l3_proto",
         "l4_proto",
         "midstream",
-        "thread_ts_msec",
+        "thread_ts_usec",
         "src_ip",
         "dst_ip"
     ],
@@ -36,11 +37,10 @@
     },
 
     "if": {
-        "properties": { "flow_event_name": { "enum": [ "guessed", "detected",
-                                                       "detection-update", "not-detected" ] } }
+        "properties": { "flow_event_name": { "enum": [ "analyse" ] } }
     },
     "then": {
-        "required": [ "ndpi" ]
+        "required": [ "data_analysis" ]
     },
 
     "if": {
@@ -50,6 +50,14 @@
         "required": [ "ndpi" ]
     },
 
+    "if": {
+        "properties": { "flow_event_name": { "enum": [ "guessed", "detected",
+                                                       "detection-update", "not-detected" ] } }
+    },
+    "then": {
+        "required": [ "ndpi" ]
+    },
+
     "properties": {
         "alias": {
             "type": "string"
@@ -69,7 +77,7 @@
         "flow_event_id": {
             "type": "number",
             "minimum": 0,
-            "maximum": 8
+            "maximum": 9
         },
         "flow_event_name": {
             "type": "string",
@@ -79,6 +87,7 @@
                 "end",
                 "idle",
                 "update",
+                "analyse",
                 "guessed",
                 "detected",
                 "detection-update",
@@ -117,7 +126,11 @@
             "type": "number",
             "minimum": 0
         },
-        "flow_last_seen": {
+        "flow_src_last_pkt_time": {
+            "type": "number",
+            "minimum": 0
+        },
+        "flow_dst_last_pkt_time": {
             "type": "number",
             "minimum": 0
         },
@@ -178,7 +191,7 @@
             "minimum": 0,
             "maximum": 1
         },
-        "thread_ts_msec": {
+        "thread_ts_usec": {
             "type": "number",
             "minimum": 0
         },
@@ -328,6 +341,88 @@
                 }
             },
             "additionalProperties": false
+        },
+        "data_analysis": {
+            "type": "object",
+            "required": [ "iat", "pktlen" ],
+
+            "properties": {
+                "iat": {
+                    "type": "object",
+
+                    "properties": {
+                        "flow_min": {
+                            "type": "number"
+                        },
+                        "flow_avg": {
+                            "type": "number"
+                        },
+                        "flow_max": {
+                            "type": "number"
+                        },
+                        "flow_stddev": {
+                            "type": "number"
+                        },
+                        "c_to_s_min": {
+                            "type": "number"
+                        },
+                        "c_to_s_avg": {
+                            "type": "number"
+                        },
+                        "c_to_s_max": {
+                            "type": "number"
+                        },
+                        "c_to_s_stddev": {
+                            "type": "number"
+                        },
+                        "s_to_c_min": {
+                            "type": "number"
+                        },
+                        "s_to_c_avg": {
+                            "type": "number"
+                        },
+                        "s_to_c_max": {
+                            "type": "number"
+                        },
+                        "s_to_c_stddev": {
+                            "type": "number"
+                        }
+                    },
+                    "additionalProperties": false
+                },
+                "pktlen": {
+                    "type": "object",
+
+                    "properties": {
+                        "c_to_s_min": {
+                            "type": "number"
+                            },
+                        "c_to_s_avg": {
+                            "type": "number"
+                        },
+                        "c_to_s_max": {
+                            "type": "number"
+                        },
+                        "c_to_s_stddev": {
+                            "type": "number"
+                        },
+                        "s_to_c_min": {
+                            "type": "number"
+                        },
+                        "s_to_c_avg": {
+                            "type": "number"
+                        },
+                        "s_to_c_max": {
+                            "type": "number"
+                        },
+                        "s_to_c_stddev": {
+                            "type": "number"
+                        }
+                    },
+                    "additionalProperties": false
+                }
+            },
+            "additionalProperties": false
         }
     },
     "additionalProperties": false
diff --git a/schema/packet_event_schema.json b/schema/packet_event_schema.json
index e0e549d71..f38d8aa58 100644
--- a/schema/packet_event_schema.json
+++ b/schema/packet_event_schema.json
@@ -13,21 +13,21 @@
         "pkt_l4_offset",
         "pkt_len",
         "pkt_l4_len",
-        "thread_ts_msec"
+        "thread_ts_usec"
     ],
 
     "dependencies" : {
-        "flow_id" : [ "flow_packet_id", "flow_last_seen", "flow_idle_time" ]
+        "flow_id" : [ "flow_packet_id", "flow_src_last_pkt_time", "flow_dst_last_pkt_time", "flow_idle_time" ]
     },
 
     "if": {
         "properties": { "packet_event_name": { "enum": ["packet-flow"] } }
     },
     "then": {
-        "required": [ "thread_id", "flow_id", "flow_packet_id", "flow_last_seen", "flow_idle_time" ]
+        "required": [ "thread_id", "flow_id", "flow_packet_id", "flow_src_last_pkt_time", "flow_dst_last_pkt_time", "flow_idle_time" ]
     },
     "else": {
-        "not": { "required": [ "thread_id", "flow_id", "flow_packet_id", "flow_last_seen", "flow_idle_time" ] }
+        "not": { "required": [ "thread_id", "flow_id", "flow_packet_id", "flow_src_last_pkt_time", "flow_dst_last_pkt_time", "flow_idle_time" ] }
     },
 
     "properties": {
@@ -64,7 +64,11 @@
         "flow_packet_id": {
             "type": "number"
         },
-        "flow_last_seen": {
+        "flow_src_last_pkt_time": {
+            "type": "number",
+            "minimum": 0
+        },
+        "flow_dst_last_pkt_time": {
             "type": "number",
             "minimum": 0
         },
@@ -95,7 +99,7 @@
             "minimum": 0,
             "maximum": 65535
         },
-        "thread_ts_msec": {
+        "thread_ts_usec": {
             "type": "number",
             "minimum": 0
         },