Major nDPId extension. Sorry for the huge commit.

 - nDPId: fixed invalid IP4/IP6 tuple compare
 - nDPIsrvd: fixed caching issue (finally)
 - added tiny c example (can be used to check flow manager sanity)
 - c-captured: use flow_last_seen timestamp from `struct nDPIsrvd_flow`
 - README.md update: added example JSON sequence
 - nDPId: added new flow event `update` necessary for correct
   timeout handling (and other future use-cases)
 - nDPIsrvd.h and nDPIsrvd.py: switched to an instance
   (consists of an alias/source tuple) based flow manager
 - every flow related event **must** now serialize `alias`, `source`,
   `flow_id`, `flow_last_seen` and `flow_idle_time` to make the timeout
   handling and verification process work correctly
 - nDPIsrvd.h: ability to profile any dynamic memory (de-)allocation
 - nDPIsrvd.py: removed PcapPacket class (unused)
 - py-flow-dashboard and py-flow-multiprocess: fixed race condition
 - py-flow-info: print statusbar with probably useful information
 - nDPId/nDPIsrvd.h: switched from packet-flow only timestamps (`pkt_*sec`)
   to a generic flow event timestamp `ts_msec`
 - nDPId-test: added additional checks
 - nDPId: increased ICMP flow timeout
 - nDPId: using event based i/o if capturing packets from a device
 - nDPIsrvd: fixed memory leak on shutdown if remote descriptors
   were still connected

Signed-off-by: Toni Uhlig <matzeton@googlemail.com>

3 files changed, 98 insertions, 21 deletions

diff --git a/schema/daemon_event_schema.json b/schema/daemon_event_schema.json
index a22e57f61..72b0b61a2 100644
--- a/schema/daemon_event_schema.json
+++ b/schema/daemon_event_schema.json
@@ -8,6 +8,18 @@
         "daemon_event_id",
         "daemon_event_name"
     ],
+    "if": {
+        "properties": { "daemon_event_name": { "const": "init" } }
+    },
+    "then": {
+        "required": [ "max-flows-per-thread", "max-idle-flows-per-thread", "tick-resolution", "reader-thread-count", "flow-scan-interval", "generic-max-idle-time", "icmp-max-idle-time", "udp-max-idle-time", "tcp-max-idle-time", "max-packets-per-flow-to-send", "max-packets-per-flow-to-process" ]
+    },
+    "if": {
+        "properties": { "daemon_event_name": { "const": "shutdown" } }
+    },
+    "then": {
+        "required": [ "total-events-serialized" ]
+    },
     "properties": {
         "alias": {
             "type": "string"
@@ -50,7 +62,7 @@
         "reader-thread-count": {
             "type": "number"
         },
-        "idle-scan-period": {
+        "flow-scan-interval": {
             "type": "number"
         },
         "generic-max-idle-time": {
@@ -65,14 +77,15 @@
         "tcp-max-idle-time": {
             "type": "number"
         },
-        "tcp-max-post-end-flow-time": {
-            "type": "number"
-        },
         "max-packets-per-flow-to-process": {
             "type": "number"
         },
         "max-packets-per-flow-to-send": {
             "type": "number"
+        },
+        "total-events-serialized": {
+            "type": "number",
+            "minimum": 1
         }
     },
     "additionalProperties": false
diff --git a/schema/flow_event_schema.json b/schema/flow_event_schema.json
index 11c105879..65ef899fc 100644
--- a/schema/flow_event_schema.json
+++ b/schema/flow_event_schema.json
@@ -8,9 +8,10 @@
         "flow_event_id",
         "flow_event_name",
         "flow_id",
-        "flow_packet_id",
+        "flow_packets_processed",
         "flow_first_seen",
         "flow_last_seen",
+        "flow_idle_time",
         "flow_min_l4_payload_len",
         "flow_max_l4_payload_len",
         "flow_tot_l4_payload_len",
@@ -18,9 +19,40 @@
         "l3_proto",
         "l4_proto",
         "midstream",
+        "ts_msec",
         "src_ip",
         "dst_ip"
     ],
+    "if": {
+        "properties": { "flow_event_name": { "const": "update" } }
+    },
+    "then": {
+        "required": [ "flow_datalink", "flow_max_packets" ]
+    },
+    "if": {
+        "properties": { "flow_event_name": { "const": "not-detected" } }
+    },
+    "then": {
+        "required": [ "ndpi" ]
+    },
+    "if": {
+        "properties": { "flow_event_name": { "const": "guessed" } }
+    },
+    "then": {
+        "required": [ "ndpi" ]
+    },
+    "if": {
+        "properties": { "flow_event_name": { "const": "detected" } }
+    },
+    "then": {
+        "required": [ "ndpi" ]
+    },
+    "if": {
+        "properties": { "flow_event_name": { "const": "detection-update" } }
+    },
+    "then": {
+        "required": [ "ndpi" ]
+    },
     "properties": {
         "alias": {
             "type": "string"
@@ -40,7 +72,7 @@
         "flow_event_id": {
             "type": "number",
             "minimum": 0,
-            "maximum": 7
+            "maximum": 8
         },
         "flow_event_name": {
             "type": "string",
@@ -49,24 +81,29 @@
                 "new",
                 "end",
                 "idle",
+                "update",
                 "guessed",
                 "detected",
                 "detection-update",
                 "not-detected"
             ]
         },
+        "flow_id": {
+            "type": "number",
+            "minimum": 1
+        },
         "flow_datalink": {
             "type": "number",
             "minimum": 0,
             "maximum": 265
         },
-        "flow_id": {
-            "type": "number",
-            "minimum": 1
-        },
-        "flow_packet_id": {
+        "flow_packets_processed": {
             "type": "number"
         },
+        "flow_max_packets": {
+            "type": "number",
+            "minimum": 0
+        },
         "flow_first_seen": {
             "type": "number",
             "minimum": 24710
@@ -75,8 +112,9 @@
             "type": "number",
             "minimum": 24710
         },
-        "flow_max_packets": {
-            "type": "number"
+        "flow_idle_time": {
+            "type": "number",
+            "minimum": 1
         },
         "flow_min_l4_payload_len": {
             "type": "number"
@@ -119,6 +157,10 @@
             "minimum": 0,
             "maximum": 1
         },
+        "ts_msec": {
+            "type": "number",
+            "minimum": 0
+        },
         "src_ip": {
             "type": "string",
             "anyOf" : [
diff --git a/schema/packet_event_schema.json b/schema/packet_event_schema.json
index abdeb7721..0b53ae99c 100644
--- a/schema/packet_event_schema.json
+++ b/schema/packet_event_schema.json
@@ -6,8 +6,25 @@
         "thread_id",
         "packet_id",
         "packet_event_id",
-        "packet_event_name"
+        "packet_event_name",
+        "pkt_oversize",
+        "pkt_caplen",
+        "pkt_type",
+        "pkt_l3_offset",
+        "pkt_l4_offset",
+        "pkt_len",
+        "pkt_l4_len",
+        "ts_msec"
     ],
+    "dependencies" : {
+        "flow_id" : [ "flow_packet_id", "flow_last_seen", "flow_idle_time" ]
+    },
+    "if": {
+        "properties": { "packet_event_name": { "const": "packet-flow" } }
+    },
+    "then": {
+        "required": [ "flow_id" ]
+    },
     "properties": {
         "alias": {
             "type": "string"
@@ -42,6 +59,14 @@
         "flow_packet_id": {
             "type": "number"
         },
+        "flow_last_seen": {
+            "type": "number",
+            "minimum": 24710
+        },
+        "flow_idle_time": {
+            "type": "number",
+            "minimum": 1
+        },
         "pkt_caplen": {
             "type": "number",
             "minimum": 1,
@@ -65,6 +90,10 @@
             "minimum": 0,
             "maximum": 65535
         },
+        "ts_msec": {
+            "type": "number",
+            "minimum": 0
+        },
         "pkt_l4_offset": {
             "type": "number",
             "minimum": 0,
@@ -75,13 +104,6 @@
             "minimum": 0,
             "maximum": 65535
         },
-        "pkt_ts_usec": {
-            "type": "number"
-        },
-        "pkt_ts_sec": {
-            "type": "number",
-            "minimum": 1
-        },
         "pkt": {
             "type": "string"
         }