Added JSON schema files and a Python schema validator.

Signed-off-by: Toni Uhlig <matzeton@googlemail.com>

6 files changed, 465 insertions, 1 deletions

diff --git a/dependencies/nDPIsrvd.py b/dependencies/nDPIsrvd.py
index 53ed131e8..056557792 100644
--- a/dependencies/nDPIsrvd.py
+++ b/dependencies/nDPIsrvd.py
@@ -9,6 +9,7 @@ import os
 import scapy.all
 import stat
 import socket
+import sys
 try:
     from colorama import Back, Fore, Style
     USE_COLORAMA=True
@@ -178,7 +179,7 @@ class nDPIsrvdSocket:
         while True:
             if self.receive() > 0:
                 if self.parse(callback, global_user_data) is False:
-                    sys.stderr.write('Callback returned False, abort.\n')
+                    raise RuntimeError('Callback returned False, abort.')
                     break;
 
 class PcapPacket:
@@ -317,3 +318,29 @@ def validateAddress(args):
         address = address_tcpip
 
     return address
+
+global schema
+schema = {'packet_event_schema' : None, 'basic_event_schema' : None, 'daemon_event_schema' : None, 'flow_event_schema' : None}
+
+def initSchemaValidator(schema_dir='./schema'):
+    for key in schema:
+        with open(schema_dir + '/' + str(key) + '.json', 'r') as schema_file:
+            schema[key] = json.load(schema_file)
+
+def validateAgainstSchema(json_dict):
+    import jsonschema
+
+    if 'packet_event_id' in json_dict:
+        jsonschema.validate(instance=json_dict, schema=schema['packet_event_schema'])
+        return True
+    if 'basic_event_id' in json_dict:
+        jsonschema.validate(instance=json_dict, schema=schema['basic_event_schema'])
+        return True
+    if 'daemon_event_id' in json_dict:
+        jsonschema.validate(instance=json_dict, schema=schema['daemon_event_schema'])
+        return True
+    if 'flow_event_id' in json_dict:
+        jsonschema.validate(instance=json_dict, schema=schema['flow_event_schema'])
+        return True
+
+    return False
diff --git a/examples/py-schema-validation/py-schema-validation.py b/examples/py-schema-validation/py-schema-validation.py
new file mode 100755
index 000000000..407c7b814
--- /dev/null
+++ b/examples/py-schema-validation/py-schema-validation.py
@@ -0,0 +1,41 @@
+#!/usr/bin/env python3
+
+import os
+import sys
+
+sys.path.append(os.path.dirname(sys.argv[0]) + '/../../dependencies')
+import nDPIsrvd
+from nDPIsrvd import nDPIsrvdSocket, TermColor
+
+class Stats:
+    lines_processed = 0
+    print_dot_every = 10
+    next_lines_print = print_dot_every
+
+def onJsonLineRecvd(json_dict, current_flow, global_user_data):
+    validation_done = nDPIsrvd.validateAgainstSchema(json_dict)
+
+    global_user_data.lines_processed += 1
+    if global_user_data.lines_processed % global_user_data.print_dot_every == 0:
+        sys.stdout.write('.')
+        sys.stdout.flush()
+    if global_user_data.lines_processed == global_user_data.next_lines_print:
+        global_user_data.next_lines_print *= 2
+        sys.stdout.write(str(global_user_data.lines_processed))
+        sys.stdout.flush()
+
+    return validation_done
+
+if __name__ == '__main__':
+    argparser = nDPIsrvd.defaultArgumentParser()
+    args = argparser.parse_args()
+    address = nDPIsrvd.validateAddress(args)
+
+    sys.stderr.write('Recv buffer size: {}\n'.format(nDPIsrvd.NETWORK_BUFFER_MAX_SIZE))
+    sys.stderr.write('Connecting to {} ..\n'.format(address[0]+':'+str(address[1]) if type(address) is tuple else address))
+
+    nDPIsrvd.initSchemaValidator(os.path.dirname(sys.argv[0]) + '/../../schema')
+
+    nsock = nDPIsrvdSocket()
+    nsock.connect(address)
+    nsock.loop(onJsonLineRecvd, Stats())
diff --git a/schema/basic_event_schema.json b/schema/basic_event_schema.json
new file mode 100644
index 000000000..626602ad2
--- /dev/null
+++ b/schema/basic_event_schema.json
@@ -0,0 +1,82 @@
+{
+    "type": "object",
+    "required": [
+        "alias",
+        "source",
+        "thread_id",
+        "packet_id",
+        "basic_event_id",
+        "basic_event_name"
+    ],
+    "properties": {
+        "alias": {
+            "type": "string"
+        },
+        "source": {
+            "type": "string"
+        },
+        "thread_id": {
+            "type": "number"
+        },
+        "packet_id": {
+            "type": "number"
+        },
+        "basic_event_id": {
+            "type": "number",
+            "minimum": 0,
+            "maximum": 16
+        },
+        "basic_event_name": {
+            "type": "string"
+        },
+        "datalink": {
+            "type": "number",
+            "minimum": 0,
+            "maximum": 265
+        },
+        "header": {
+            "type": "number"
+        },
+        "type": {
+            "type": "number",
+            "minimum": 0,
+            "maximum": 65535
+        },
+        "protocol": {
+            "type": "number",
+            "minimum": 0,
+            "maximum": 65535
+        },
+        "caplen": {
+            "type": "number"
+        },
+        "len": {
+            "type": "number"
+        },
+        "ip_size": {
+            "type": "number"
+        },
+        "expected": {
+            "type": "number"
+        },
+        "l4_data_len": {
+            "type": "number"
+        },
+        "header_len": {
+            "type": "number"
+        },
+        "size": {
+            "type": "number"
+        },
+        "current_active": {
+            "type": "number"
+        },
+        "current_idle": {
+            "type": "number"
+        },
+        "max_active": {
+            "type": "number"
+        }
+    },
+    "additionalProperties": false
+}
diff --git a/schema/daemon_event_schema.json b/schema/daemon_event_schema.json
new file mode 100644
index 000000000..020b55161
--- /dev/null
+++ b/schema/daemon_event_schema.json
@@ -0,0 +1,67 @@
+{
+    "type": "object",
+    "required": [
+        "alias",
+        "source",
+        "thread_id",
+        "packet_id",
+        "daemon_event_id",
+        "daemon_event_name"
+    ],
+    "properties": {
+        "alias": {
+            "type": "string"
+        },
+        "source": {
+            "type": "string"
+        },
+        "thread_id": {
+            "type": "number",
+            "minimum": 0,
+            "maximum": 31
+        },
+        "packet_id": {
+            "type": "number",
+            "minimum": 1
+        },
+        "daemon_event_id": {
+            "type": "number",
+            "minimum": 0,
+            "maximum": 3
+        },
+        "daemon_event_name": {
+            "type": "string",
+            "enum": [
+                "invalid",
+                "init",
+                "reconnect",
+                "shutdown"
+            ]
+        },
+        "max-flows-per-thread": {
+            "type": "number"
+        },
+        "max-idle-flows-per-thread": {
+            "type": "number"
+        },
+        "tick-resolution": {
+            "type": "number"
+        },
+        "reader-thread-count": {
+            "type": "number"
+        },
+        "idle-scan-period": {
+            "type": "number"
+        },
+        "max-idle-time": {
+            "type": "number"
+        },
+        "tcp-max-post-end-flow-time": {
+            "type": "number"
+        },
+        "max-packets-per-flow-to-send": {
+            "type": "number"
+        }
+    },
+    "additionalProperties": false
+}
diff --git a/schema/flow_event_schema.json b/schema/flow_event_schema.json
new file mode 100644
index 000000000..6ed7c4615
--- /dev/null
+++ b/schema/flow_event_schema.json
@@ -0,0 +1,159 @@
+{
+    "type": "object",
+    "required": [
+        "alias",
+        "source",
+        "thread_id",
+        "packet_id",
+        "flow_event_id",
+        "flow_event_name",
+        "flow_id",
+        "flow_packet_id",
+        "flow_first_seen",
+        "flow_last_seen",
+        "flow_min_l4_data_len",
+        "flow_max_l4_data_len",
+        "flow_tot_l4_data_len",
+        "flow_avg_l4_data_len",
+        "l3_proto",
+        "l4_proto",
+        "midstream",
+        "src_ip",
+        "dst_ip"
+    ],
+    "properties": {
+        "alias": {
+            "type": "string"
+        },
+        "source": {
+            "type": "string"
+        },
+        "thread_id": {
+            "type": "number",
+            "minimum": 0,
+            "maximum": 31
+        },
+        "packet_id": {
+            "type": "number",
+            "minimum": 1
+        },
+        "flow_event_id": {
+            "type": "number",
+            "minimum": 0,
+            "maximum": 7
+        },
+        "flow_event_name": {
+            "type": "string",
+            "enum": [
+                "invalid",
+                "new",
+                "end",
+                "idle",
+                "guessed",
+                "detected",
+                "detection-update",
+                "not-detected"
+            ]
+        },
+        "flow_datalink": {
+            "type": "number",
+            "minimum": 0,
+            "maximum": 265
+        },
+        "flow_id": {
+            "type": "number",
+            "minimum": 1
+        },
+        "flow_packet_id": {
+            "type": "number"
+        },
+        "flow_first_seen": {
+            "type": "number"
+        },
+        "flow_last_seen": {
+            "type": "number"
+        },
+        "flow_max_packets": {
+            "type": "number"
+        },
+        "flow_min_l4_data_len": {
+            "type": "number"
+        },
+        "flow_max_l4_data_len": {
+            "type": "number"
+        },
+        "flow_tot_l4_data_len": {
+            "type": "number"
+        },
+        "flow_avg_l4_data_len": {
+            "type": "number"
+        },
+        "l3_proto": {
+            "type": "string",
+            "enum": [
+                "ip4",
+                "ip6",
+                "unknown"
+            ]
+        },
+        "l4_proto": {
+            "type": "string",
+            "oneOf": [
+                {
+                    "pattern": "[0-9]+"
+                },
+                {
+                    "enum": [
+                        "tcp",
+                        "udp",
+                        "icmp",
+                        "icmp6"
+                    ]
+                }
+            ]
+        },
+        "midstream": {
+            "type": "number",
+            "minimum": 0,
+            "maximum": 1
+        },
+        "src_ip": {
+            "type": "string"
+        },
+        "dst_ip": {
+            "type": "string"
+        },
+        "src_port": {
+            "type": "number",
+            "minimum": 1,
+            "maximum": 65535
+        },
+        "dst_port": {
+            "type": "number",
+            "minimum": 1,
+            "maximum": 65535
+        },
+        "ndpi": {
+            "type": "object"
+        },
+        "tls": {
+            "type": "object"
+        },
+        "quic": {
+            "type": "object"
+        },
+        "http": {
+            "type": "object"
+        },
+        "smtp": {
+            "type": "object"
+        },
+        "dns": {
+            "type": "object"
+        },
+        "ssh": {
+            "type": "object"
+        }
+    },
+    "additionalProperties": false
+}
diff --git a/schema/packet_event_schema.json b/schema/packet_event_schema.json
new file mode 100644
index 000000000..d3f5a77d9
--- /dev/null
+++ b/schema/packet_event_schema.json
@@ -0,0 +1,88 @@
+{
+    "type": "object",
+    "required": [
+        "alias",
+        "source",
+        "thread_id",
+        "packet_id",
+        "packet_event_id",
+        "packet_event_name"
+    ],
+    "properties": {
+        "alias": {
+            "type": "string"
+        },
+        "source": {
+            "type": "string"
+        },
+        "thread_id": {
+            "type": "number"
+        },
+        "packet_id": {
+            "type": "number"
+        },
+        "packet_event_id": {
+            "type": "number",
+            "minimum": 0,
+            "maximum": 2
+        },
+        "packet_event_name": {
+            "type": "string",
+            "enum": [
+                "invalid",
+                "packet",
+                "packet-flow"
+            ]
+        },
+        "flow_id": {
+            "type": "number",
+            "minimum": 1
+        },
+        "flow_packet_id": {
+            "type": "number"
+        },
+        "pkt_caplen": {
+            "type": "number",
+            "minimum": 1,
+            "maximum": 65535
+        },
+        "pkt_type": {
+            "type": "number",
+            "minimum": 0,
+            "maximum": 65535
+        },
+        "pkt_oversize": {
+            "type": "boolean"
+        },
+        "pkt_l3_offset": {
+            "type": "number",
+            "minimum": 0,
+            "maximum": 65535
+        },
+        "pkt_l4_len": {
+            "type": "number",
+            "minimum": 0,
+            "maximum": 65535
+        },
+        "pkt_l4_offset": {
+            "type": "number",
+            "minimum": 0,
+            "maximum": 65535
+        },
+        "pkt_len": {
+            "type": "number",
+            "minimum": 0,
+            "maximum": 65535
+        },
+        "pkt_ts_usec": {
+            "type": "number"
+        },
+        "pkt_ts_sec": {
+            "type": "number"
+        },
+        "pkt": {
+            "type": "string"
+        }
+    },
+    "additionalProperties": false
+}