Complete rework of nDPI Python bindings (cffi API, automatic generation, packaging and CI integration)

3 files changed, 259 insertions, 0 deletions

diff --git a/python/ndpi/__init__.py b/python/ndpi/__init__.py
new file mode 100644
index 000000000..f6f5b9079
--- /dev/null
+++ b/python/ndpi/__init__.py
@@ -0,0 +1,21 @@
+"""
+------------------------------------------------------------------------------------------------------------------------
+__init__.py
+Copyright (C) 2011-22 - ntop.org
+This file is part of nDPI, an open source deep packet inspection library.
+nDPI is free software: you can redistribute it and/or modify it under the terms of the GNU Lesser General Public
+License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later
+version.
+nDPI is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty
+of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more details.
+You should have received a copy of the GNU Lesser General Public License along with NFStream.
+If not, see <http://www.gnu.org/licenses/>.
+------------------------------------------------------------------------------------------------------------------------
+"""
+
+from .ndpi import NDPI, NDPIFlow
+
+
+__author__ = """Zied Aouini"""
+__email__ = 'aouinizied@gmail.com'
+__version__ = '4.3.0'
diff --git a/python/ndpi/ndpi.py b/python/ndpi/ndpi.py
new file mode 100644
index 000000000..0dd5b56cf
--- /dev/null
+++ b/python/ndpi/ndpi.py
@@ -0,0 +1,101 @@
+"""
+------------------------------------------------------------------------------------------------------------------------
+ndpi.py
+Copyright (C) 2011-22 - ntop.org
+This file is part of nDPI, an open source deep packet inspection library.
+nDPI is free software: you can redistribute it and/or modify it under the terms of the GNU Lesser General Public
+License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later
+version.
+nDPI is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty
+of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more details.
+You should have received a copy of the GNU Lesser General Public License along with NFStream.
+If not, see <http://www.gnu.org/licenses/>.
+------------------------------------------------------------------------------------------------------------------------
+"""
+
+from collections import namedtuple
+from _ndpi import ffi, lib
+
+
+ndpi_protocol = namedtuple('NDPIProtocol', ['C',
+                                            'master_protocol',
+                                            'app_protocol',
+                                            'category'])
+
+ndpi_confidence = namedtuple('NDPIConfidence', ['id',
+                                                'name'])
+
+
+class NDPI(object):
+    __slots__ = ("_api_version",
+                 "_revision",
+                 "_detection_module")
+
+    def __init__(self):
+        self._detection_module = lib.ndpi_init_detection_module(0)
+        if self._detection_module == ffi.NULL:
+            raise MemoryError("Unable to instantiate NDPI object")
+        lib.ndpi_py_setup_detection_module(self._detection_module)
+
+    @property
+    def api_version(self):
+        return lib.ndpi_get_api_version()
+
+    @property
+    def revision(self):
+        return ffi.string(lib.ndpi_revision()).decode('utf-8', errors='ignore')
+
+    def process_packet(self, flow, packet, packet_time_ms):
+        p = lib.ndpi_detection_process_packet(self._detection_module,
+                                              flow._C,
+                                              packet,
+                                              len(packet),
+                                              int(packet_time_ms))
+        return ndpi_protocol(C=p,
+                             master_protocol=p.master_protocol,
+                             app_protocol=p.app_protocol,
+                             category=p.category)
+
+    def giveup(self, flow, enable_guess=True):
+        p = lib.ndpi_detection_giveup(self._detection_module,
+                                      flow._C,
+                                      enable_guess,
+                                      ffi.new("uint8_t*", 0))
+        return ndpi_protocol(C=p,
+                             master_protocol=p.master_protocol,
+                             app_protocol=p.app_protocol,
+                             category=p.category)
+
+    def protocol_name(self, protocol):
+        buf = ffi.new("char[40]")
+        lib.ndpi_protocol2name(self._detection_module, protocol.C, buf, ffi.sizeof(buf))
+        return ffi.string(buf).decode('utf-8', errors='ignore')
+
+    def protocol_category_name(self, protocol):
+        return ffi.string(lib.ndpi_category_get_name(self._detection_module,
+                                                     protocol.C.category)).decode('utf-8',
+                                                                                  errors='ignore')
+
+    def __del__(self):
+        if self._detection_module != ffi.NULL:
+            lib.ndpi_exit_detection_module(self._detection_module)
+
+
+class NDPIFlow(object):
+    __slots__ = ("_C")
+
+    @property
+    def confidence(self):
+        confidence = self._C.confidence
+        return ndpi_confidence(id=confidence,
+                               name=ffi.string(lib.ndpi_confidence_get_name(confidence)).decode('utf-8',
+                                                                                                errors='ignore'))
+
+    def __init__(self):
+        self._C = lib.ndpi_py_initialize_flow()
+
+    def __del__(self):
+        if self._C != ffi.NULL:
+            lib.ndpi_flow_free(self._C)
+            self._C = ffi.NULL
+
diff --git a/python/ndpi/ndpi_build.py b/python/ndpi/ndpi_build.py
new file mode 100644
index 000000000..8bc412126
--- /dev/null
+++ b/python/ndpi/ndpi_build.py
@@ -0,0 +1,137 @@
+"""
+------------------------------------------------------------------------------------------------------------------------
+ndpi_build.py
+Copyright (C) 2011-22 - ntop.org
+This file is part of nDPI, an open source deep packet inspection library.
+nDPI is free software: you can redistribute it and/or modify it under the terms of the GNU Lesser General Public
+License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later
+version.
+nDPI is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty
+of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more details.
+You should have received a copy of the GNU Lesser General Public License along with NFStream.
+If not, see <http://www.gnu.org/licenses/>.
+------------------------------------------------------------------------------------------------------------------------
+"""
+
+from cffi import FFI
+import subprocess
+import pathlib
+
+NDPI_INCLUDES = """
+#include "ndpi_main.h"
+#include "ndpi_typedefs.h"
+#include "ndpi_api.h"
+"""
+
+NDPI_HELPERS = """
+// nDPI cffi helper functions (function naming convention ndpi_py_*)
+
+void ndpi_py_setup_detection_module(struct ndpi_detection_module_struct *mod) {
+  if (mod == NULL) {
+    return;
+  } else {
+    NDPI_PROTOCOL_BITMASK protos;
+    NDPI_BITMASK_SET_ALL(protos); // Set bitmask for ALL protocols
+    ndpi_set_protocol_detection_bitmask2(mod, &protos);
+    ndpi_finalize_initialization(mod);
+  }
+};
+
+struct ndpi_flow_struct * ndpi_py_initialize_flow(void) {
+  struct ndpi_flow_struct * ndpi_flow = NULL;
+  ndpi_flow = (struct ndpi_flow_struct *)ndpi_flow_malloc(SIZEOF_FLOW_STRUCT);
+  memset(ndpi_flow, 0, SIZEOF_FLOW_STRUCT);
+  return ndpi_flow;
+};
+"""
+
+
+NDPI_APIS = """
+u_int16_t ndpi_get_api_version(void);
+char* ndpi_revision(void);
+struct ndpi_detection_module_struct *ndpi_init_detection_module(ndpi_init_prefs prefs);
+void ndpi_exit_detection_module(struct ndpi_detection_module_struct *ndpi_struct);
+void ndpi_flow_free(void *ptr);
+ndpi_protocol ndpi_detection_process_packet(struct ndpi_detection_module_struct *ndpi_struct,
+                                            struct ndpi_flow_struct *flow,
+                                            const unsigned char *packet,
+                                            const unsigned short packetlen,
+                                            const u_int64_t packet_time_ms);
+ndpi_protocol ndpi_detection_giveup(struct ndpi_detection_module_struct *ndpi_struct,
+                                    struct ndpi_flow_struct *flow,
+                                    u_int8_t enable_guess,
+                                    u_int8_t *protocol_was_guessed);
+void ndpi_py_setup_detection_module(struct ndpi_detection_module_struct *mod);
+struct ndpi_flow_struct * ndpi_py_initialize_flow(void);
+char* ndpi_protocol2name(struct ndpi_detection_module_struct *ndpi_mod, ndpi_protocol proto, char *buf, u_int buf_len);
+const char* ndpi_category_get_name(struct ndpi_detection_module_struct *ndpi_mod, ndpi_protocol_category_t category);
+const char* ndpi_confidence_get_name(ndpi_confidence_t confidence);
+"""
+
+ffi_builder = FFI()
+
+
+INCLUDE_DIR = pathlib.Path(__file__)\
+    .parent.resolve().parent.resolve().parent.resolve().\
+    joinpath("src").joinpath("include")
+
+LIBRARY_DIR = pathlib.Path(__file__)\
+    .parent.resolve().parent.resolve().parent.resolve().\
+    joinpath("src").joinpath("lib")
+
+
+NDPI_CDEF = subprocess.run(["gcc",
+                            "-DNDPI_LIB_COMPILATION",
+                            "-DNDPI_CFFI_PREPROCESSING",
+                            "-DNDPI_CFFI_PREPROCESSING_EXCLUDE_PACKED",
+                            "-E", "-x", "c", "-P", "-C",
+                            str(INCLUDE_DIR.joinpath("ndpi_typedefs.h"))],
+                           capture_output=True
+                           ).stdout.decode('utf-8',
+                                           errors='ignore')
+
+NDPI_MODULE_STRUCT_CDEF = NDPI_CDEF.split("//CFFI.NDPI_MODULE_STRUCT")[1]
+
+
+NDPI_PACKED = subprocess.run(["gcc",
+                              "-DNDPI_LIB_COMPILATION", "-DNDPI_CFFI_PREPROCESSING",
+                              "-E", "-x", "c", "-P", "-C",
+                              str(INCLUDE_DIR.joinpath("ndpi_typedefs.h"))],
+                             capture_output=True
+                             ).stdout.decode('utf-8',
+                                             errors='ignore')
+
+NDPI_PACKED_STRUCTURES = NDPI_PACKED.split("//CFFI.NDPI_PACKED_STRUCTURES")[1]
+
+NDPI_SOURCE = NDPI_INCLUDES + NDPI_MODULE_STRUCT_CDEF + NDPI_HELPERS
+
+
+ffi_builder.set_source("_ndpi",
+                       NDPI_SOURCE,
+                       libraries=["ndpi"],
+                       library_dirs=[str(LIBRARY_DIR)],
+                       include_dirs=[str(INCLUDE_DIR)])
+
+
+ffi_builder.cdef("""
+typedef uint64_t u_int64_t;
+typedef uint32_t u_int32_t;
+typedef uint16_t u_int16_t;
+typedef uint8_t u_int8_t;
+typedef uint8_t u_char;
+typedef unsigned u_int;
+struct in_addr {
+  unsigned long s_addr;
+};
+struct in6_addr {
+    unsigned char s6_addr[16];
+};
+""")
+
+ffi_builder.cdef(NDPI_PACKED_STRUCTURES, packed=True)
+ffi_builder.cdef(NDPI_CDEF)
+ffi_builder.cdef(NDPI_APIS)
+
+
+if __name__ == "__main__":
+    ffi_builder.compile(verbose=True)