diff options
| -rw-r--r-- | configure.seed | 2 | ||||
| -rw-r--r-- | python/Makefile.in | 25 | ||||
| -rw-r--r-- | python/README | 7 | ||||
| -rwxr-xr-x | python/ndpi_example.py | 252 | ||||
| -rw-r--r-- | python/ndpi_typestruct.py | 773 | ||||
| -rw-r--r-- | python/ndpi_wrap.c | 55 |
6 files changed, 1113 insertions, 1 deletions
diff --git a/configure.seed b/configure.seed index 1f66bf460..a3cc646f2 100644 --- a/configure.seed +++ b/configure.seed @@ -141,7 +141,7 @@ AC_ARG_ENABLE([debug-messages], AC_CHECK_LIB(pthread, pthread_setaffinity_np, AC_DEFINE_UNQUOTED(HAVE_PTHREAD_SETAFFINITY_NP, 1, [libc has pthread_setaffinity_np])) -AC_CONFIG_FILES([Makefile example/Makefile example/Makefile.dpdk tests/Makefile libndpi.pc src/include/ndpi_define.h src/lib/Makefile]) +AC_CONFIG_FILES([Makefile example/Makefile example/Makefile.dpdk tests/Makefile libndpi.pc src/include/ndpi_define.h src/lib/Makefile python/Makefile]) AC_CONFIG_HEADERS(src/include/ndpi_config.h) AC_SUBST(GIT_RELEASE) AC_SUBST(NDPI_MAJOR) diff --git a/python/Makefile.in b/python/Makefile.in new file mode 100644 index 000000000..a87ee4c2d --- /dev/null +++ b/python/Makefile.in @@ -0,0 +1,25 @@ +CC=@CC@ +CFLAGS=-I. -I../src/include -I./src/lib/third_party/include -shared -Wl, +LIBNDPI=../src/lib/libndpi.so.@NDPI_VERSION_SHORT@ +LDFLAGS=$(CFILE) $(LIBNDPI) -lpcap +SHARE = -soname,ndpi_wrap +SO=ndpi_wrap.so +OBJS = ndpi_wrap.o +CFILE = ndpi_wrap.c +PIC = -fPIC +PREFIX?=/usr/local +.PHONY: all + +UNAME_S := $(shell uname -s) +ifeq ($(UNAME_S),Darwin)#do something for linux + SHARE=-install_name,ndpiReader.so +endif + +all: $(SO) + +$(SO): $(CFILE) $(LIBNDPI) + $(CC) $(CFLAGS)$(SHARE) -o $@ $(PIC) $(LDFLAGS) + ln -s $(LIBNDPI) . + +clean: + /bin/rm -f $(SO) diff --git a/python/README b/python/README new file mode 100644 index 000000000..61a689a77 --- /dev/null +++ b/python/README @@ -0,0 +1,7 @@ +This direcgory contains the Python bindings for nDPI + +Usage: +- python3 ndpi_example.py <interface> +- python3 ndpi_example.py <pcap_file> + +Code courtesy of Massimo Puddu diff --git a/python/ndpi_example.py b/python/ndpi_example.py new file mode 100755 index 000000000..fb0135c46 --- /dev/null +++ b/python/ndpi_example.py @@ -0,0 +1,252 @@ +#!/usr/bin/env python3 +# +# ndpi_example.py +# +# Copyright (C) 2011-19 - ntop.org +# +# nDPI is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# nDPI is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with nDPI. If not, see <http://www.gnu.org/licenses/>. +# + +from ndpi_typestruct import * +from scapy.all import * + + +# ------- return type & pcapstruct to declare ------- + + +class workflow(Structure): + _fields_ = [("src_ip", c_uint32), + ("dst_ip", c_uint32), + ("src_port", c_uint16), + ("dst_port", c_uint16), + ("protocol", c_uint8), + ("packets", c_uint32), + ("detected_protocol", ndpi_protocol), + ("detection_completed", c_uint8), + ("id", c_uint32), + ("src_id", POINTER(ndpi_id_struct)), + ("dst_id", POINTER(ndpi_id_struct)), + ("flow", POINTER(ndpi_flow_struct))] + + +CMCFUN = CFUNCTYPE(c_int, c_void_p, c_void_p) +GUESS = CFUNCTYPE(None, c_void_p, c_int32, c_int, c_void_p) +FREE = CFUNCTYPE(None, c_void_p) + + +def node_proto_guess_walker(nodo, which, depth, user_data): + global ndpi_info_mod + + flow = cast(nodo, POINTER(POINTER(workflow))).contents.contents + if which == 0 or which == 3: #execute only preorder operation of the tree + if flow.detection_completed == 0: # order for tree operation + flow.detected_protocol = ndpi.ndpi_detection_giveup(ndpi_info_mod, flow.flow, flow.protocol, + int(socket.ntohl(flow.src_ip)), 1) + count_protocol[flow.detected_protocol.app_protocol] += flow.packets + + +def py_cmp_fun(a, b): + fa = cast(a, POINTER(workflow)) + fb = cast(b, POINTER(workflow)) + + if fa.contents.src_ip < fb.contents.src_ip: return -1 + elif fa.contents.src_ip > fb.contents.src_ip: return 1 + if fa.contents.src_port < fb.contents.src_port: return -1 + elif fa.contents.src_port > fb.contents.src_port: return 1 + if fa.contents.dst_ip < fb.contents.dst_ip: return -1 + elif fa.contents.dst_ip > fb.contents.dst_ip: return 1 + if fa.contents.dst_port < fb.contents.dst_port: return -1 + elif fa.contents.dst_port > fb.contents.dst_port: return 1 + if fa.contents.protocol < fb.contents.protocol: return -1 + elif fa.contents.protocol > fb.contents.protocol: return 1 + return 0 + + +def freer(a): + pass + + +cmp_func = CMCFUN(py_cmp_fun) +guess_walker = GUESS(node_proto_guess_walker) +free_walk = FREE(freer) + +# -------------------------------------- + +# number of anylized packet +packet_number = 0 +flow_count = 0 +max_num_udp_dissected_pkts = 16 +max_num_tcp_dissected_pkts = 10 +flows_root = c_void_p(None) +flows_root_ref = pointer(flows_root) +count_protocol = (c_int32 * (ndpi.ndpi_wrap_ndpi_max_supported_protocols() + ndpi.ndpi_wrap_ndpi_max_num_custom_protocols() + 1))() +lista = [] # used to avoid impropriate memory deallocation from python + + +# check ndpi version +if ndpi.ndpi_get_api_version() != ndpi.ndpi_wrap_get_api_version(): + print("nDPI Library version mismatch: please make sure this code and the nDPI library are in sync\n") + sys.exit(-1) + +# create data structure of ndpi +ndpi_info_mod = ndpi.ndpi_init_detection_module() +if ndpi_info_mod is None: + sys.exit(-1) + + +def ip2int(ip): + """ + Convert an IP string to long and then c_uint32 + """ + packedIP = socket.inet_aton(ip) + return int(struct.unpack("!I", packedIP)[0]) + + +def get_flow(packet): + global flows_root + global flows_root_ref + global flow_count + + ip_packet = packet[1] + ip_protocol = ip_packet.proto + transport_packet = None + + if ip_protocol == 6 or ip_protocol == 17: transport_packet = packet[2] + if transport_packet is not None: + # to avoid two nodes in one binary tree for a flow + ip_src = ip2int(ip_packet.src) + ip_dst = ip2int(ip_packet.dst) + src_port = transport_packet.sport + dst_port = transport_packet.dport + else: return None + # set flows to correct type and data + ndpi_flow = pointer(ndpi_flow_struct()) + memset(ndpi_flow, 0, sizeof(ndpi_flow_struct)) + if ip_src > ip_dst: + flow = workflow(ip_src, ip_dst, src_port, dst_port, int(ip_packet.proto), 0, ndpi_protocol(), 0, 0, pointer(ndpi_id_struct()), pointer(ndpi_id_struct()), ndpi_flow) + else: + flow = workflow(ip_dst, ip_src, dst_port, src_port, int(ip_packet.proto), 0, ndpi_protocol(), 0, 0, pointer(ndpi_id_struct()), pointer(ndpi_id_struct()), ndpi_flow) + flow_ref = pointer(flow) + res = ndpi.ndpi_tfind(flow_ref, flows_root_ref, cmp_func) + if res is None: + ndpi.ndpi_tsearch(flow_ref, pointer(flows_root), cmp_func) # add + lista.append(flow) + flow_count += 1 + return pointer(flow) + flow = cast(res, POINTER(POINTER(workflow))).contents + return flow + + +def packetcaptured(packet): + global packet_number + global start_time + global ndpi_info_mod + + flow = None + h = pcap_pkthdr() + + #getting flow + try: + flow = get_flow(packet) + except AttributeError: + pass # ignore packet + if flow is None: return + + #filling pcap_pkthdr + h.len = h.caplen = len(packet) + h.ts.tv_sec = int(packet["IP"].time/1000000) + h.ts.tv_usec = round(packet["IP"].time) + + # real work + if int(packet[1].frag) == 0: # not fragmented packet + flow.contents.packets += 1 + packet_number += 1 + # get ndpi_iphdr address + iphdr_addr = cast(c_char_p(packet[1].build()), c_void_p) + ndpi_flow = flow.contents.flow + + if flow.contents.detection_completed is 0: + flow.contents.detected_protocol = ndpi.ndpi_detection_process_packet(ndpi_info_mod, ndpi_flow, cast(iphdr_addr, POINTER(c_uint8)), + int(packet[1].len), h.ts.tv_usec, flow.contents.src_id, flow.contents.dst_id) + + flow1 = flow.contents.detected_protocol + + valid = False + + if flow.contents.protocol == 6: valid = flow.contents.packets > max_num_tcp_dissected_pkts + elif flow.contents.protocol == 17: valid = flow.contents.packets > max_num_udp_dissected_pkts + + #should we continue anylizing packet or not? + if valid or flow1.app_protocol is not 0: + if valid or flow1.master_protocol is not 91: #or # 91 is NDPI_PROTOCOL_TLS + flow.contents.detection_completed = 1 #protocol found + if flow1.app_protocol is 0: + flow.contents.detected_protocol = ndpi.ndpi_detection_giveup(ndpi_info_mod, ndpi_flow, 1) + + + +def result(): + global flows_root_ref + global ndpi_info_mod + print('\nnumber of anylized packet ' + str(packet_number)) + print('number of flows ' + str(flow_count)) + + ndpi.ndpi_twalk(flows_root_ref.contents, guess_walker, None) + + print('\nDetected protocols:') + for i in range(0, ndpi.ndpi_get_num_supported_protocols(ndpi_info_mod)): + if count_protocol[i] > 0: + print(cast(ndpi.ndpi_get_proto_name(ndpi_info_mod, i), c_char_p).value + ': '.encode('UTF-8') + str(count_protocol[i]).encode('UTF-8')) + + +def free(ndpi_struct): + ndpi.ndpi_tdestroy(flows_root, free_walk) + ndpi.ndpi_exit_detection_module(ndpi_struct) + + +def initialize(ndpi_struct): + all = NDPI_PROTOCOL_BITMASK() + ndpi.ndpi_wrap_NDPI_BITMASK_SET_ALL(pointer(all)) + ndpi.ndpi_set_protocol_detection_bitmask2(ndpi_struct, pointer(all)) + + +print('Using nDPI ' + str(cast(ndpi.ndpi_revision(), c_char_p).value)) + +initialize(ndpi_info_mod) + +if(len(sys.argv) != 2): + print("\nUsage: ndpi_example.py <device>") + sys.exit(0) + +if "." in sys.argv[1]: + print('Reading pcap from file ' + sys.argv[1] + '...') + scapy_cap = None + try: + scapy_cap = rdpcap(sys.argv[1]) + except FileNotFoundError: + print("\nFile not found") + except Scapy_Exception: + print("\nBad pcap") + else: + for packet in scapy_cap: + packetcaptured(packet) +else: + print('Capturing live traffic from device ' + sys.argv[1] + '...') + try: + sniff(iface=sys.argv[1], prn=packetcaptured) + except KeyboardInterrupt: + print('\nInterrupted\n') + +result() +free(ndpi_info_mod) diff --git a/python/ndpi_typestruct.py b/python/ndpi_typestruct.py new file mode 100644 index 000000000..942db32d5 --- /dev/null +++ b/python/ndpi_typestruct.py @@ -0,0 +1,773 @@ +# +# ndpi_typestruct.h +# +# Copyright (C) 2011-18 - ntop.org +# +# nDPI is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# nDPI is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with nDPI. If not, see <http://www.gnu.org/licenses/>. +# + + +import os +from ctypes import * + +ndpi = CDLL('./ndpi_wrap.so') + +# NDPI_SELECTION_BITMASK_PROTOCOL_SIZE = c_uint32 +# ndpi_protocol_category_t, ndpi_protocol_breed_t e ndpi_log_level_t sono enumeratori e vengono impostati come c_int + +class ndpi_detection_module_struct(Structure): + pass + +class ndpi_flow_struct(Structure): + pass + +class ndpi_protocol(Structure): + _fields_ = [ + ("master_protocol", c_uint16), + ("app_protocol", c_uint16), + ("category", c_int) + + ] + +class timeval(Structure): + _fields_ = [("tv_sec", c_ulong), ("tv_usec", c_ulong)] + +class pcap_pkthdr(Structure): + _fields_ = [("ts", timeval), ("caplen", c_uint32), ("len", c_uint32)] + +#dal file ../src/include/ndpi_tydedefs.h +class ndpi_ndpi_mask(Structure): + _fields_ = [("fds_bits", c_uint32)] + +class NDPI_PROTOCOL_BITMASK(Structure): + _fields_ = [("fds_bits", ndpi_ndpi_mask * ndpi.ndpi_wrap_ndpi_num_fds_bits())] + +class ndpi_subprotocol_conf_struct(Structure): + _fields_ = [("func", CFUNCTYPE(c_void_p,POINTER(ndpi_detection_module_struct),c_char_p,c_char_p,c_int))] + +class ndpi_automa(Structure): + _fields_ = [ + ("ac_automa", c_void_p), #Real type is AC_AUTOMATA_t + ("ac_automa_finalized", c_uint8) + ] + +class struct_node_t(Structure): + pass +struct_node_t._fields_ = [ + ('key', POINTER(c_char)), + ('left', POINTER(struct_node_t)), + ('right', POINTER(struct_node_t)), +] + +class ndpi_call_function_struct(Structure): + _fields_ = [ + ("detection_bitmask", NDPI_PROTOCOL_BITMASK), + ("excluded_protocol_bitmask",NDPI_PROTOCOL_BITMASK), + ("ndpi_selection_bitmask", c_uint32), + ("func", CFUNCTYPE(None, POINTER(ndpi_detection_module_struct), POINTER(ndpi_flow_struct))), + ("detection_feature", c_uint8) + ] + +class ndpi_proto_defaults_t(Structure): + _fields_ = [ + ("protoName", POINTER(c_char)), + ("protoCategory",c_uint), + ("can_have_a_subprotocol", c_uint8), + ("protoId", c_uint16), + ("protoIdx", c_uint16), + ("master_tcp_protoId", c_uint16 * 2), + ("master_udp_protoId", c_uint16 * 2), + ("protoBreed", c_uint), + ("func", CFUNCTYPE(None, POINTER(ndpi_detection_module_struct), POINTER(ndpi_flow_struct))), + ] + +class ndpi_default_ports_tree_node_t(Structure): + _fields_ = [ + ("proto", ndpi_proto_defaults_t), + ("customUserProto",c_uint8), + ("default_port", c_int16) + ] + +# NDPI_PROTOCOL_BITTORRENT +class spinlock_t(Structure): + _fields_ = [("val", c_int)] #volatile mancante + +class atomic_t(Structure): + _fields_ = [("counter", c_int)] #volatile mancante + +class time_t(Structure): + _fields_ = [("counter", c_longlong)] #piattaforma dipendente, tuttavia รจ solo un intero + +class hash_ip4p_node(Structure): + pass + +hash_ip4p_node._fields_ = [ + ("next", POINTER(hash_ip4p_node)), + ("prev", POINTER(hash_ip4p_node)), + ("lchg", time_t), + ("port", c_uint16), + ("count", c_uint16, 12), + ("flag", c_uint16, 4), + ("ip", c_uint32) +] + +class hash_ip4p(Structure): + _fields_ = [ + ("top", POINTER(hash_ip4p_node)), + ("lock",spinlock_t), + ("len", c_size_t) + ] + +class hash_ip4p_table(Structure): + _fields_ = [ + ("size", c_size_t), + ("ipv6",c_int), + ("lock", spinlock_t), + ("count", atomic_t), + ("tbl", hash_ip4p) + ] + +class bt_announce(Structure): # 192 bytes + _fields_ = [ + ("hash", c_uint32 * 5), + ("ip", c_uint32 * 4), + ("time", c_uint32), + ("port", c_uint16), + ("name_len", c_uint8), + ("name", c_uint8 * 149) # 149 bytes + ] + +class ndpi_lru_cache(Structure): # 192 bytes + _fields_ = [ + ("num_entries", c_uint32), + ("entries", POINTER(c_uint32)), + ] + +class cache_entry(Structure): + pass + +cache_entry._fields_ = [ + ("item", c_void_p), + ("item_size", c_uint32), + ("prev", POINTER(cache_entry)), + ("next", POINTER(cache_entry)) +] + +class cache_entry_map(Structure): + pass + +cache_entry_map._fields_ = [ + ("entry", POINTER(cache_entry)), + ("next", POINTER(cache_entry_map)), +] + +class cache(Structure): # 192 bytes + _fields_ = [ + ("size", c_uint32), + ("max_size", c_uint32), + ("head", POINTER(cache_entry)), + ("tail", POINTER(cache_entry)), + ("map", POINTER(POINTER(cache_entry_map))) + ] + +class custom_categories(Structure): + _fields_ =[ + #Hyperscam + #("hostnames", POINTER(hs)), + #("num_to_load", c_uint), + #("to_load", POINTER(hs_list)), + ("hostnames", ndpi_automa), + ("hostnames_shadow", ndpi_automa), + ("hostnames_hash", c_void_p), + ("ipAddresses", c_void_p), + ("ipAddresses_shadow", c_void_p), # Patricia + ("categories_loaded", c_uint8), + ] + + +ndpi_detection_module_struct._fields_ = [ + ("detection_bitmask", NDPI_PROTOCOL_BITMASK), + ("generic_http_packet_bitmask", NDPI_PROTOCOL_BITMASK), + + ("current_ts", c_uint32), + + ("ticks_per_second", c_uint32), + + #("user_data", c_void_p), debug + + ("custom_category_labels", (c_char * ndpi.ndpi_wrap_num_custom_categories()) * ndpi.ndpi_wrap_custom_category_label_len()), + + #callback function buffer + ("callback_buffer", ndpi_call_function_struct * (ndpi.ndpi_wrap_ndpi_max_supported_protocols() + 1)), + ("callback_buffer_size", c_uint32), + + ("callback_buffer_tcp_no_payload", ndpi_call_function_struct * (ndpi.ndpi_wrap_ndpi_max_supported_protocols() + 1)), + ("callback_buffer_size_tcp_no_payload", c_uint32), + + ("callback_buffer_tcp_payload", ndpi_call_function_struct * (ndpi.ndpi_wrap_ndpi_max_supported_protocols() + 1)), + ("callback_buffer_size_tcp_payload", c_uint32), + + ("callback_buffer_udp", ndpi_call_function_struct * (ndpi.ndpi_wrap_ndpi_max_supported_protocols() + 1)), + ("callback_buffer_size_udp", c_uint32), + + ("callback_buffer_non_tcp_udp", ndpi_call_function_struct * (ndpi.ndpi_wrap_ndpi_max_supported_protocols() + 1)), + ("callback_buffer_size_non_tcp_udp", c_uint32), + + ("tcpRoot", POINTER(ndpi_default_ports_tree_node_t)), + ("udpRoot", POINTER(ndpi_default_ports_tree_node_t)), + + ("ndpi_log_level", c_uint), #default error + + # ifdef NDPI_ENABLE_DEBUG_MESSAGES + #debug callback, only set whendebug is used * / + #ndpi_debug_function_ptr ndpi_debug_printf; + #const char * ndpi_debug_print_file; + #const char * ndpi_debug_print_function; + #u_int32_t ndpi_debug_print_line; + #NDPI_PROTOCOL_BITMASK debug_bitmask; + # endif + + #misc parameters + ("tcp_max_retransmission_window_size", c_uint32), + + ("directconnect_connection_ip_tick_timeout", c_uint32), + + #subprotocol registration handler + ("subprotocol_conf", ndpi_subprotocol_conf_struct * (ndpi.ndpi_wrap_ndpi_max_supported_protocols() + 1)), + + ("ndpi_num_supported_protocols", c_uint), + ("ndpi_num_custom_protocols", c_uint), + + #HTTP / DNS / HTTPS host matching * / + ("host_automa", ndpi_automa), #Used for DNS / HTTPS + ("content_automa", ndpi_automa), # Used for HTTP subprotocol_detection + ("subprotocol_automa", ndpi_automa), # Used for HTTP subprotocol_detection + ("bigrams_automa", ndpi_automa), #TOR + ("impossible_bigrams_automa", ndpi_automa), # TOR + + ("custom_categories", custom_categories), + #IP-based protocol detection + ("protocols_ptree", c_void_p), + + #irc parameters + ("irc_timeout", c_uint32), + #gnutella parameters + ("gnutella_timeout", c_uint32), + #battlefield parameters + ("battlefield_timeout", c_uint32), + # thunder parameters + ("thunder_timeout", c_uint32), + # SoulSeek parameters + ("soulseek_connection_ip_tick_timeout", c_uint32), + # rtsp parameters + ("rtsp_connection_timeout", c_uint32), + # tvants parameters + ("tvants_connection_timeout", c_uint32), + # rstp + ("orb_rstp_ts_timeout", c_uint32), + # yahoo + ("yahoo_detect_http_connections", c_uint8), + ("yahoo_lan_video_timeout", c_uint32), + ("zattoo_connection_timeout", c_uint32), + ("jabber_stun_timeout", c_uint32), + ("jabber_file_transfer_timeout", c_uint32), + + # ifdef NDPI_ENABLE_DEBUG_MESSAGES + # define NDPI_IP_STRING_SIZE 40 + #char ip_string[NDPI_IP_STRING_SIZE]; + # endif + + ("ip_version_limit", c_uint8), + #NDPI_PROTOCOL_BITTORRENT + ("bt_ht", POINTER(hash_ip4p_table)), + # ifdef NDPI_DETECTION_SUPPORT_IPV6 + ("bt6_ht", POINTER(hash_ip4p_table)), + # endif + + # BT_ANNOUNCE + ("bt_ann", POINTER(bt_announce)), + ("bt_ann_len", c_int), + + # NDPI_PROTOCOL_OOKLA + ("ookla_cache", POINTER(ndpi_lru_cache)), + + # NDPI_PROTOCOL_TINC + ("tinc_cache", POINTER(cache)), + + ("proto_defaults", ndpi_proto_defaults_t * (ndpi.ndpi_wrap_ndpi_max_supported_protocols() + ndpi.ndpi_wrap_ndpi_max_num_custom_protocols())), + + ("http_dont_dissect_response", c_uint8, 1), + ("dns_dont_dissect_response", c_uint8, 1), + ("direction_detect_disable", c_uint8, 1), # disable internal detection of packet direction + ("disable_metadata_export", c_uint8, 1), # No metadata is exported + ("enable_category_substring_match", c_uint8, 1), # Default is perfect match + + ("hyperscan", c_void_p) # Intel Hyperscan + ] + +class u6_addr(Union): # 128-bit IP6 address + _fields_ = [ + ("u6_addr8",c_uint8 * 16), + ("u6_addr16",c_uint16 * 8), + ("u6_addr32",c_uint32 * 4) + ] + +class ndpi_in6_addr(Structure): + _fields_ = [("u6_addr", u6_addr)] + + +class ndpi_ip_addr_t(Union): + _fields_ = [ + ('ipv4', c_uint32), + ('ipv4_u_int8_t', c_uint8 * 4), + ('ipv6', ndpi_in6_addr), +] + +class ndpi_id_struct(Structure): + _fields_ = [ + ('detected_protocol_bitmask', NDPI_PROTOCOL_BITMASK), + ('rtsp_ip_address', ndpi_ip_addr_t), + ('yahoo_video_lan_timer', c_uint32), + ('irc_port', c_uint16 * 8), + ('last_time_port_used', c_uint32 * 8), + ('irc_ts', c_uint32), + ('gnutella_ts', c_uint32), + ('battlefield_ts', c_uint32), + ('thunder_ts', c_uint32), + ('rtsp_timer', c_uint32), + ('oscar_last_safe_access_time', c_uint32), + ('zattoo_ts', c_uint32), + ('jabber_stun_or_ft_ts', c_uint32), + ('directconnect_last_safe_access_time', c_uint32), + ('soulseek_last_safe_access_time', c_uint32), + ('detected_directconnect_port', c_uint16), + ('detected_directconnect_udp_port', c_uint16), + ('detected_directconnect_ssl_port', c_uint16), + ('bt_port_t', c_uint16 * 8), + ('bt_port_u', c_uint16 * 8), + ('jabber_voice_stun_port', c_uint16 * 6), + ('jabber_file_transfer_port', c_uint16 * 2), + ('detected_gnutella_port', c_uint16), + ('detected_gnutella_udp_port1', c_uint16), + ('detected_gnutella_udp_port2', c_uint16), + ('soulseek_listen_port', c_uint16), + ('irc_number_of_port', c_uint8), + ('oscar_ssl_session_id', c_uint8 * 33), + ('jabber_voice_stun_used_ports', c_uint8), + ('yahoo_video_lan_dir', c_uint32, 1), + ('yahoo_conf_logged_in', c_uint32, 1), + ('yahoo_voice_conf_logged_in', c_uint32, 1), + ('rtsp_ts_set', c_uint32, 1), +] + +#struct flow +class ndpi_flow_tcp_struct(Structure): + _fields_ = [ + # NDPI_PROTOCOL_MAIL_SMTP + ('smtp_command_bitmask', c_uint16), + ('pop_command_bitmask', c_uint16), + ('qq_nxt_len', c_uint16), + ('wa_matched_so_far', c_uint8), + ('tds_login_version', c_uint8), + ('irc_stage', c_uint8), + ('irc_port', c_uint8), + ('h323_valid_packets', c_uint8), + ('gnutella_msg_id', c_uint8 * 3), + ('irc_3a_counter', c_uint32, 3), + ('irc_stage2', c_uint32, 5), + ('irc_direction', c_uint32, 2), + ('irc_0x1000_full', c_uint32, 1), + ('soulseek_stage', c_uint32, 2), + ('tds_stage', c_uint32, 3), + ('usenet_stage', c_uint32, 2), + ('imesh_stage', c_uint32, 4), + ('http_setup_dir', c_uint32, 2), + ('http_stage', c_uint32, 2), + ('http_empty_line_seen', c_uint32, 1), + ('http_wait_for_retransmission', c_uint32, 1), + ('gnutella_stage', c_uint32, 2), + ('mms_stage', c_uint32, 2), + ('yahoo_sip_comm', c_uint32, 1), + ('yahoo_http_proxy_stage', c_uint32, 2), + ('msn_stage', c_uint32, 3), + ('msn_ssl_ft', c_uint32, 2), + ('ssh_stage', c_uint32, 3), + ('vnc_stage', c_uint32, 2), + ('telnet_stage', c_uint32, 2), + ('ssl_seen_client_cert', c_uint8, 1), + ('ssl_seen_server_cert', c_uint8, 1), + ('ssl_seen_certificate', c_uint8, 1), + ('ssl_stage', c_uint8, 2), + ('postgres_stage', c_uint32, 3), + ('ddlink_server_direction', c_uint32, 1), + ('seen_syn', c_uint32, 1), + ('seen_syn_ack', c_uint32, 1), + ('seen_ack', c_uint32, 1), + ('icecast_stage', c_uint32, 1), + ('dofus_stage', c_uint32, 1), + ('fiesta_stage', c_uint32, 2), + ('wow_stage', c_uint32, 2), + ('veoh_tv_stage', c_uint32, 2), + ('shoutcast_stage', c_uint32, 2), + ('rtp_special_packets_seen', c_uint32, 1), + ('mail_pop_stage', c_uint32, 2), + ('mail_imap_stage', c_uint32, 3), + ('mail_imap_starttls', c_uint32, 2), + ('skype_packet_id', c_uint8), + ('citrix_packet_id', c_uint8), + ('lotus_notes_packet_id', c_uint8), + ('teamviewer_stage', c_uint8), + ('prev_zmq_pkt_len', c_uint8), + ('prev_zmq_pkt', c_ubyte * 10), + ('ppstream_stage', c_uint32, 3), + ('memcached_matches', c_uint8), + ('nest_log_sink_matches', c_uint8), +] + +class ndpi_flow_udp_struct(Structure): + _fields_ = [ + ('battlefield_msg_id', c_uint32), + ('snmp_msg_id', c_uint32), + ('battlefield_stage', c_uint32, 3), + ('snmp_stage', c_uint32, 2), + ('ppstream_stage', c_uint32, 3), + ('halflife2_stage', c_uint32, 2), + ('tftp_stage', c_uint32, 1), + ('aimini_stage', c_uint32, 5), + ('xbox_stage', c_uint32, 1), + ('wsus_stage', c_uint32, 1), + ('skype_packet_id', c_uint8), + ('teamviewer_stage', c_uint8), + ('eaq_pkt_id', c_uint8), + ('eaq_sequence', c_uint32), + ('rx_conn_epoch', c_uint32), + ('rx_conn_id', c_uint32), + ('memcached_matches', c_uint8), + ] + +# the tcp / udp / other l4 value union used to reduce the number of bytes for tcp or udp protocol states +class l4(Union): + _fields_ = [("tcp", ndpi_flow_tcp_struct),("udp", ndpi_flow_udp_struct)] + +class http(Structure): + _fields_ = [ + ("method", c_int), + ("url", c_char_p), + ("content_type", c_char_p), + ("num_request_headers", c_uint8), ("num_response_headers", c_uint8), + ("request_version", c_uint8), # 0=1.0 and 1=1.1. Create an enum for this? + ("response_status_code", c_uint16), # 200, 404, etc. + ] + +class dns(Structure): # the only fields useful for nDPI and ntopng + _fields_ = [ + ("num_queries", c_uint8), ("num_answers", c_uint8), ("reply_code", c_uint8), + ("query_type", c_uint16), ("query_class", c_uint16), ("rsp_type", c_uint16), + ("rsp_addr", ndpi_ip_addr_t) # The first address in a DNS response packet + ] + +class ntp(Structure): + _fields_ = [("request_code", c_uint8), ("version", c_uint8)] + +class ssl(Structure): + _fields_ = [ + ("ssl_version", c_uint8), + ("client_certificate", c_char * 64), ("server_certificate", c_char * 64), ("server_organization", c_char * 64), + ("ja3_client", c_char * 33), ("ja3_server", c_char * 33), + ("server_cipher", c_uint16), + ("server_unsafe_cipher", c_int) + ] + +class stun(Structure): + _fields_ = [ + ("num_udp_pkts", c_uint8), + ("num_processed_pkts", c_uint8), + ("num_binding_requests", c_uint8), + ("is_skype", c_uint8) + ] + +class stun_ssl(Union): # We can have STUN over SSL thus they need to live together + _fields_ = [("ssl", ssl),("stun",stun)] + +class ssh(Structure): + _fields_ = [("client_signature", c_char * 48), ("server_signature", c_char * 48)] + +class mdns(Structure): + _fields_ = [("answer", c_char * 96)] + +class ubntac2(Structure): + _fields_ = [("version", c_char * 96)] + +class http2(Structure): + _fields_ = [ + ("detected_os", c_ubyte * 32), #Via HTTP User-Agent + ("nat_ip", c_ubyte * 24) + ] + +class bittorrent(Structure): # Bittorrent hash + _fields_ = [ ("hash", c_ubyte * 20) ] + +class dhcp(Structure): + _fields_ = [ + ("fingerprint", c_char * 48), + ("nat_ip", c_char * 48) + ] + +class protos(Union): + _fields_ = [ + ("dns", dns), + ("ntp", ntp), + ("stun_ssl", stun_ssl), + ("ssh", ssh), + ("mdns", mdns), + ("ubntac2", ubntac2), + ("http", http2), + ("bittorrent", bittorrent), + ("dhcp", dhcp) + ] + +class tinc_cache_entry(Structure): + _fields_ = [ + ('src_address', c_uint32), + ('dst_address', c_uint32), + ('dst_port', c_uint16), +] + +class struct_ndpi_int_one_line_struct(Structure): + _fields_ = [ + ('ptr', POINTER(c_uint8)), + ('len', c_uint16), +] + +class struct_ndpi_iphdr_little_end(Structure): + _fields_ = [ + ('ihl', c_uint8, 4), + ('version', c_uint8, 4), + ('tos', c_uint8), + ('tot_len', c_uint16), + ('id', c_uint16), + ('frag_off', c_uint16), + ('ttl', c_uint8), + ('protocol', c_uint8), + ('check', c_uint16), + ('saddr', c_uint32), + ('daddr', c_uint32)] + +class struct_ndpi_ip6_hdrctl(Structure): + _fields_ = [ + ('ip6_un1_flow', c_uint32), + ('ip6_un1_plen', c_uint16), + ('ip6_un1_nxt', c_uint8), + ('ip6_un1_hlim', c_uint8), +] + +class struct_ndpi_ipv6hdr(Structure): + _fields_ = [ + ('ip6_hdr', struct_ndpi_ip6_hdrctl), + ('ip6_src', ndpi_in6_addr), + ('ip6_dst', ndpi_in6_addr), +] + +class struct_ndpi_tcphdr(Structure): + _fields_ = [ + ('source', c_uint16), + ('dest', c_uint16), + ('seq', c_uint32), + ('ack_seq', c_uint32), + ('res1', c_uint16, 4), + ('doff', c_uint16, 4), + ('fin', c_uint16, 1), + ('syn', c_uint16, 1), + ('rst', c_uint16, 1), + ('psh', c_uint16, 1), + ('ack', c_uint16, 1), + ('urg', c_uint16, 1), + ('ece', c_uint16, 1), + ('cwr', c_uint16, 1), + ('window', c_uint16), + ('check', c_uint16), + ('urg_ptr', c_uint16), +] + +class struct_ndpi_udphdr(Structure): + _fields_ = [ + ('source', c_uint16), + ('dest', c_uint16), + ('len', c_uint16), + ('check', c_uint16), +] + +class ndpi_packet_struct(Structure): + _fields_ = [ + ('iph', POINTER(struct_ndpi_iphdr_little_end)), + ('iphv6', POINTER(struct_ndpi_ipv6hdr)), + ('tcp', POINTER(struct_ndpi_tcphdr)), + ('udp', POINTER(struct_ndpi_udphdr)), + ('generic_l4_ptr', POINTER(c_uint8)), + ('payload', POINTER(c_uint8)), + ('tick_timestamp', c_uint32), + ('tick_timestamp_l', c_uint64), + ('detected_protocol_stack', c_uint16 * 2), + ('detected_subprotocol_stack', c_uint8 * 2), + ('protocol_stack_info', c_uint16), + ('line', struct_ndpi_int_one_line_struct * 64), + ('host_line', struct_ndpi_int_one_line_struct), + ('forwarded_line', struct_ndpi_int_one_line_struct), + ('referer_line', struct_ndpi_int_one_line_struct), + ('content_line', struct_ndpi_int_one_line_struct), + ('accept_line', struct_ndpi_int_one_line_struct), + ('user_agent_line', struct_ndpi_int_one_line_struct), + ('http_url_name', struct_ndpi_int_one_line_struct), + ('http_encoding', struct_ndpi_int_one_line_struct), + ('http_transfer_encoding', struct_ndpi_int_one_line_struct), + ('http_contentlen', struct_ndpi_int_one_line_struct), + ('http_cookie', struct_ndpi_int_one_line_struct), + ('http_origin', struct_ndpi_int_one_line_struct), + ('http_x_session_type', struct_ndpi_int_one_line_struct), + ('server_line', struct_ndpi_int_one_line_struct), + ('http_method', struct_ndpi_int_one_line_struct), + ('http_response', struct_ndpi_int_one_line_struct), + ('http_num_headers', c_uint8), + ('l3_packet_len', c_uint16), + ('l4_packet_len', c_uint16), + ('payload_packet_len', c_uint16), + ('actual_payload_len', c_uint16), + ('num_retried_bytes', c_uint16), + ('parsed_lines', c_uint16), + ('parsed_unix_lines', c_uint16), + ('empty_line_position', c_uint16), + ('tcp_retransmission', c_uint8), + ('l4_protocol', c_uint8), + ('ssl_certificate_detected', c_uint8, 4), + ('ssl_certificate_num_checks', c_uint8, 4), + ('packet_lines_parsed_complete', c_uint8, 1), + ('packet_direction', c_uint8, 1), + ('empty_line_position_set', c_uint8, 1), +] + +ndpi_flow_struct._fields_ = [ + ("detected_protocol_stack", c_uint16 * ndpi.ndpi_wrap_ndpi_procol_size()), + ("protocol_stack_info", c_uint16), + + # init parameter, internal used to set up timestamp,... + ("guessed_protocol_id", c_uint16), + ("guessed_host_protocol_id", c_uint16), + ("guessed_category", c_uint16), + ("guessed_header_category", c_uint16), + ("protocol_id_already_guessed", c_uint8, 1), + ("host_already_guessed", c_uint8, 1), + ("init_finished", c_uint8, 1), + ("setup_packet_direction", c_uint8, 1), + ("packet_direction", c_uint8, 1), + ("check_extra_packets", c_uint8, 1), + + # if ndpi_struct->direction_detect_disable == 1 tcp sequence number connection tracking + ("next_tcp_seq_nr", c_uint32 * 2), + + ("max_extra_packets_to_check", c_uint8), + ("num_extra_packets_checked", c_uint8), + ("num_processed_pkts", c_uint8), # <= WARNING it can wrap but we do expect people to giveup earlier + + ("extra_packets_func", CFUNCTYPE(c_int,POINTER(ndpi_detection_module_struct),POINTER(ndpi_flow_struct))), + + ("l4", l4), + + # Pointer to src or dst that identifies the server of this connection + ("server_id", ndpi_id_struct), + # HTTP host or DNS query + ("host_server_name", c_ubyte * 256), + + + # This structure below will not not stay inside the protos + # structure below as HTTP is used by many subprotocols + # such as FaceBook, Google... so it is hard to know + # when to use it or not. Thus we leave it outside for the + # time being. + + + ("http", http), + ("protos", protos), + + # ALL protocol specific 64 bit variables here + + # protocols which have marked a connection as this connection cannot be protocol XXX, multiple u_int64_t + ("excluded_protocol_bitmask", NDPI_PROTOCOL_BITMASK), + + ("category", c_int), + + ('redis_s2d_first_char', c_uint8), + ('redis_d2s_first_char', c_uint8), + ('packet_counter', c_uint16), + ('packet_direction_counter', c_uint16 * 2), + ('byte_counter', c_uint16 * 2), + ('bittorrent_stage', c_uint8), + ('directconnect_stage', c_uint8, 2), + ('sip_yahoo_voice', c_uint8, 1), + ('http_detected', c_uint8, 1), + ('http_upper_protocol', c_uint16), + ('http_lower_protocol', c_uint16), + ('rtsprdt_stage', c_uint8, 2), + ('rtsp_control_flow', c_uint8, 1), + ('yahoo_detection_finished', c_uint8, 2), + ('zattoo_stage', c_uint8, 3), + ('qq_stage', c_uint8, 3), + ('thunder_stage', c_uint8, 2), + ('oscar_ssl_voice_stage', c_uint8, 3), + ('oscar_video_voice', c_uint8, 1), + ('florensia_stage', c_uint8, 1), + ('socks5_stage', c_uint8, 2), + ('socks4_stage', c_uint8, 2), + ('edonkey_stage', c_uint8, 2), + ('ftp_control_stage', c_uint8, 2), + ('rtmp_stage', c_uint8, 2), + ('pando_stage', c_uint8, 3), + ('steam_stage1', c_uint16, 3), + ('steam_stage2', c_uint16, 2), + ('steam_stage3', c_uint16, 2), + ('pplive_stage1', c_uint8, 3), + ('pplive_stage2', c_uint8, 2), + ('pplive_stage3', c_uint8, 2), + ('starcraft_udp_stage', c_uint8, 3), + ('ovpn_session_id', c_uint8 * 8), + ('ovpn_counter', c_uint8), + ('tinc_state', c_uint8), + ('tinc_cache_entry', tinc_cache_entry), + ('csgo_strid', c_uint8 * 18), + ('csgo_state', c_uint8), + ('csgo_s2', c_uint8), + ('csgo_id2', c_uint32), + ('kxun_counter', c_uint16), + ('iqiyi_counter', c_uint16), + ('packet', ndpi_packet_struct), + ('flow', POINTER(ndpi_flow_struct)), + ('src', POINTER(ndpi_id_struct)), + ('dst', POINTER(ndpi_id_struct)) +] + + +ndpi.ndpi_tfind.restype = c_void_p +ndpi.ndpi_tsearch.restype = c_void_p +ndpi.ndpi_revision.restype = c_void_p +ndpi.ndpi_get_proto_name.restype = c_void_p +ndpi.ndpi_get_num_supported_protocols.restype = c_uint +ndpi.ndpi_detection_process_packet.restype = ndpi_protocol +ndpi.ndpi_init_detection_module.restype = POINTER(ndpi_detection_module_struct) +ndpi.ndpi_wrap_NDPI_BITMASK_SET_ALL.argtypes = [POINTER(NDPI_PROTOCOL_BITMASK)] +ndpi.ndpi_set_protocol_detection_bitmask2.argtypes = [POINTER(ndpi_detection_module_struct), POINTER(NDPI_PROTOCOL_BITMASK)] +ndpi.ndpi_tsearch.argtypes = [c_void_p, POINTER(c_void_p), CFUNCTYPE(c_int, c_void_p, c_void_p)] +ndpi.ndpi_twalk.argtypes = [c_void_p, CFUNCTYPE(None, c_void_p, c_int32, c_int, c_void_p), c_void_p] +ndpi.ndpi_tdestroy.argtypes = [c_void_p, CFUNCTYPE(None, c_void_p)] +ndpi.ndpi_detection_giveup.restype = ndpi_protocol +ndpi.ndpi_detection_giveup.argtypes = [POINTER(ndpi_detection_module_struct), POINTER(ndpi_flow_struct), c_uint8] +ndpi.ndpi_detection_process_packet.argtypes = [POINTER(ndpi_detection_module_struct), POINTER(ndpi_flow_struct), POINTER(c_ubyte), c_ushort, c_uint64, POINTER(ndpi_id_struct), POINTER(ndpi_id_struct)] diff --git a/python/ndpi_wrap.c b/python/ndpi_wrap.c new file mode 100644 index 000000000..c8f7fdd96 --- /dev/null +++ b/python/ndpi_wrap.c @@ -0,0 +1,55 @@ +/* + * ndpi_wrap.c + * + * Copyright (C) 2011-19 - ntop.org + * + * nDPI is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * nDPI is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with nDPI. If not, see <http://www.gnu.org/licenses/>. + * + */ + +#include "sys/types.h" +#include "ndpi_config.h" +#include "ndpi_main.h" + +int ndpi_wrap_get_api_version(){ + return NDPI_API_VERSION; +} + +int ndpi_wrap_ndpi_num_fds_bits(){ + return NDPI_NUM_FDS_BITS; +} + +int ndpi_wrap_num_custom_categories(){ + return NUM_CUSTOM_CATEGORIES; +} + +int ndpi_wrap_custom_category_label_len(){ + return CUSTOM_CATEGORY_LABEL_LEN; +} + +int ndpi_wrap_ndpi_max_supported_protocols(){ + return NDPI_MAX_SUPPORTED_PROTOCOLS; +} + +int ndpi_wrap_ndpi_max_num_custom_protocols(){ + return NDPI_MAX_NUM_CUSTOM_PROTOCOLS; +} + +int ndpi_wrap_ndpi_procol_size(){ + return NDPI_PROTOCOL_SIZE; +} + +void ndpi_wrap_NDPI_BITMASK_SET_ALL(NDPI_PROTOCOL_BITMASK* bitmask){ + NDPI_ONE(bitmask); +}
\ No newline at end of file |