diff options
Diffstat (limited to 'python')
| -rw-r--r-- | python/Makefile.in | 27 | ||||
| -rw-r--r-- | python/README.md | 79 | ||||
| -rw-r--r-- | python/README.rst | 43 | ||||
| -rw-r--r-- | python/dev_requirements.txt | 2 | ||||
| -rw-r--r-- | python/flow_printer.py | 38 | ||||
| -rw-r--r-- | python/ndpi.py | 1346 | ||||
| -rw-r--r-- | python/ndpi/__init__.py | 21 | ||||
| -rw-r--r-- | python/ndpi/ndpi.py | 101 | ||||
| -rw-r--r-- | python/ndpi/ndpi_build.py | 137 | ||||
| -rw-r--r--[-rwxr-xr-x] | python/ndpi_example.py | 398 | ||||
| -rw-r--r-- | python/ndpi_typestruct.py | 780 | ||||
| -rw-r--r-- | python/ndpi_wrap.c | 60 | ||||
| -rw-r--r-- | python/python_extensions_guide.pdf | bin | 11854 -> 0 bytes | |||
| -rw-r--r-- | python/requirements.txt | 1 | ||||
| -rw-r--r-- | python/setup.py | 57 | ||||
| -rw-r--r-- | python/tests.py | 28 |
16 files changed, 572 insertions, 2546 deletions
diff --git a/python/Makefile.in b/python/Makefile.in deleted file mode 100644 index 3b976804a..000000000 --- a/python/Makefile.in +++ /dev/null @@ -1,27 +0,0 @@ -CC=@CC@ -CFLAGS=-I. -I../src/include -I./src/lib/third_party/include @CFLAGS@ @CUSTOM_NDPI@ -shared -#LIBNDPI=../src/lib/libndpi.so.@NDPI_VERSION_SHORT@ -LIBNDPI=../src/lib/libndpi.a -LDFLAGS=@LDFLAGS@ -LIBS=$(CFILE) $(LIBNDPI) -lpcap @LIBS@ @ADDITIONAL_LIBS@ -SHARE = -soname,ndpi_wrap -SO=ndpi_wrap.so -OBJS = ndpi_wrap.o -CFILE = ndpi_wrap.c -PIC = -fPIC -PREFIX?=/usr/local -.PHONY: all - -UNAME_S := $(shell uname -s) -ifeq ($(UNAME_S),Darwin)#do something for linux - SHARE=-install_name,ndpiReader.so -endif - -all: $(SO) - -$(SO): $(CFILE) $(LIBNDPI) Makefile - $(CC) $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $(PIC) -Wl,$(SHARE) -o $@ $(LIBS) -# ln -s $(LIBNDPI) . - -clean: - /bin/rm -f $(SO) diff --git a/python/README.md b/python/README.md new file mode 100644 index 000000000..8cb2f64c6 --- /dev/null +++ b/python/README.md @@ -0,0 +1,79 @@ +# ndpi + +This package contains Python bindings for nDPI. nDPI is an Open and Extensible LGPLv3 Deep Packet Inspection Library. + +**ndpi** is implemented using [**CFFI**][cffi] (out-of-line API mode). Consequently, it is fast and [**PyPy**][pypy] +compliant. + +## Installation + +### Build nDPI + +``` bash +git clone --branch dev https://github.com/ntop/nDPI.git +cd nDPI +./autogen.sh +./configure +make +``` + +### Install ndpi package + +``` bash +cd python +python3 -m install -r dev_requirements.txt +python3 setup.py install +``` + +## Usage + +### API +``` python +from ndpi import NDPI, NDPIFlow + +nDPI = NDPI() + +# You per flow processing here +# ... + +ndpi_flow = NDPIFlow() +nDPI.process_packet(ndpi_flow, ip_bytes, time_ms) +nDPI.giveup(ndpi_flow) # If you want to guess it instead (DPI fallback) +``` + +### Example Application + +[ndpi_example.py][ndpi_example] is provided to demonstrate how **ndpi** can be integrated within your Python application. + +``` bash +Using nDPI 4.3.0-3532-8dd70b70 +usage: ndpi_example.py [-h] [-u] input + +positional arguments: + input input pcap file path + +optional arguments: + -h, --help show this help message and exit + -u, --include-unknowns +``` + +Example with a Skype capture file + +``` bash +python ndpi_example.py -u ../tests/pcap/skype.pcap +``` + +## Related projects + +The provided example is for demo purposes only, For additional features (live capture, multiplatform support, +multiprocessing, ML based classification, system visibility, etc.), please check nDPI based +framework, [**NFStream**][nfstream]. +## License + +This project is licensed under the LGPLv3 License - see the [**License**][license] file for details. + +[license]: https://github.com/ntop/nDPI/blob/dev/COPYING +[cffi]: https://cffi.readthedocs.io/en/latest/ +[pypy]: https://www.pypy.org/ +[nfstream]: https://github.com/nfstream/nfstream +[ndpi_example]: https://github.com/ntop/nDPI/blob/dev/python/ndpi_example.py
\ No newline at end of file diff --git a/python/README.rst b/python/README.rst deleted file mode 100644 index bb8e51aef..000000000 --- a/python/README.rst +++ /dev/null @@ -1,43 +0,0 @@ -nDPI Python bindings --------------------- - -This directory contains the Python3 bindings for nDPI. We provide both cffi and ctypes based bindings. - -**cffi bindings** - -Files: - -* ndpi.py - -Example (using `nfstream <https://github.com/aouinizied/nfstream>`_ package): - -.. code-block:: bash - - pip3 install nfstream - python3 flow_printer.py <interface> - python3 flow_printer.py <pcap_file> - -Code courtesy: - -* Zied Aouini - -**ctypes bindings** - -Files: - -* ndpi_typestruct.py -* ndpi_wrap.c -* Makefile.in - -Example: - -.. code-block:: bash - - pip3 install scapy - python3 ndpi_example.py <interface> - python3 ndpi_example.py <pcap_file> - -Code courtesy: - -* Massimo Puddu -* Zied Aouini diff --git a/python/dev_requirements.txt b/python/dev_requirements.txt new file mode 100644 index 000000000..02d5d90a3 --- /dev/null +++ b/python/dev_requirements.txt @@ -0,0 +1,2 @@ +cffi>=1.15.0 +dpkt>=1.9.7.2
\ No newline at end of file diff --git a/python/flow_printer.py b/python/flow_printer.py deleted file mode 100644 index 5579694fa..000000000 --- a/python/flow_printer.py +++ /dev/null @@ -1,38 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- - -""" ------------------------------------------------------------------------------------------------------------------------- -flow_printer.py -Copyright (C) 2019-20 - NFStream Developers -This file is part of NFStream, a Flexible Network Data Analysis Framework (https://www.nfstream.org/). -NFStream is free software: you can redistribute it and/or modify it under the terms of the GNU Lesser General Public -License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later -version. -NFStream is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty -of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details. -You should have received a copy of the GNU Lesser General Public License along with NFStream. -If not, see <http://www.gnu.org/licenses/>. ------------------------------------------------------------------------------------------------------------------------- -""" - -from nfstream import NFStreamer -import sys - -# Example must run with nfstream >= 6.1.1 -path = sys.argv[1] -flow_streamer = NFStreamer(source=path, statistical_analysis=False, performance_report=1) -result = {} -try: - for flow in flow_streamer: - print(flow) - try: - result[flow.application_name] += flow.bidirectional_packets - except KeyError: - result[flow.application_name] = flow.bidirectional_packets - print("\nSummary (Application Name: Packets):") - print(result) -except KeyboardInterrupt: - print("\nSummary (Application Name: Packets):") - print(result) - print("Terminated.") diff --git a/python/ndpi.py b/python/ndpi.py deleted file mode 100644 index ac5d1afca..000000000 --- a/python/ndpi.py +++ /dev/null @@ -1,1346 +0,0 @@ -""" -file: ndpi.py -This file is part of nfstream. - -Copyright (C) 2019-20 - nfstream.org - -nfstream is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License -as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. - -nfstream is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty -of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. - -You should have received a copy of the GNU General Public License along with nfstream. -If not, see <http://www.gnu.org/licenses/>. -""" - -from os.path import abspath, dirname -import cffi - -cc_ndpi_network_headers = """ -struct ptr_uint32 { - uint32_t value; -}; - -struct ndpi_chdlc -{ - uint8_t addr; /* 0x0F (Unicast) - 0x8F (Broadcast) */ - uint8_t ctrl; /* always 0x00 */ - uint16_t proto_code; /* protocol type (e.g. 0x0800 IP) */ -}; - -/* SLARP - Serial Line ARP http://tinyurl.com/qa54e95 */ -struct ndpi_slarp -{ - /* address requests (0x00) - address replies (0x01) - keep-alive (0x02) - */ - uint32_t slarp_type; - uint32_t addr_1; - uint32_t addr_2; -}; - -/* Cisco Discovery Protocol http://tinyurl.com/qa6yw9l */ -struct ndpi_cdp -{ - uint8_t version; - uint8_t ttl; - uint16_t checksum; - uint16_t type; - uint16_t length; -}; - -/* +++++++++++++++ Ethernet header (IEEE 802.3) +++++++++++++++ */ -struct ndpi_ethhdr -{ - uint8_t h_dest[6]; /* destination eth addr */ - uint8_t h_source[6]; /* source ether addr */ - uint16_t h_proto; /* data length (<= 1500) or type ID proto (>=1536) */ -}; - -/* +++++++++++++++ ARP header +++++++++++++++ */ -struct ndpi_arphdr { - uint16_t ar_hrd;/* Format of hardware address. */ - uint16_t ar_pro;/* Format of protocol address. */ - uint8_t ar_hln;/* Length of hardware address. */ - uint8_t ar_pln;/* Length of protocol address. */ - uint16_t ar_op;/* ARP opcode (command). */ - uint8_t arp_sha[6];/* sender hardware address */ - uint32_t arp_spa;/* sender protocol address */ - uint8_t arp_tha[6];/* target hardware address */ - uint32_t arp_tpa;/* target protocol address */ -}; - -/* +++++++++++++++ DHCP header +++++++++++++++ */ -struct ndpi_dhcphdr { - uint8_t msgType; - uint8_t htype; - uint8_t hlen; - uint8_t hops; - uint32_t xid;/* 4 */ - uint16_t secs;/* 8 */ - uint16_t flags; - uint32_t ciaddr;/* 12 */ - uint32_t yiaddr;/* 16 */ - uint32_t siaddr;/* 20 */ - uint32_t giaddr;/* 24 */ - uint8_t chaddr[16]; /* 28 */ - uint8_t sname[64]; /* 44 */ - uint8_t file[128]; /* 108 */ - uint32_t magic; /* 236 */ - uint8_t options[308]; -}; - -/* +++++++++++++++ MDNS rsp header +++++++++++++++ */ -struct ndpi_mdns_rsp_entry { - uint16_t rsp_type, rsp_class; - uint32_t ttl; - uint16_t data_len; -}; - -/* +++++++++++++++++++ LLC header (IEEE 802.2) ++++++++++++++++ */ -struct ndpi_snap_extension -{ - uint16_t oui; - uint8_t oui2; - uint16_t proto_ID; -}; - -struct ndpi_llc_header_snap -{ - uint8_t dsap; - uint8_t ssap; - uint8_t ctrl; - struct ndpi_snap_extension snap; -}; - -/* ++++++++++ RADIO TAP header (for IEEE 802.11) +++++++++++++ */ -struct ndpi_radiotap_header -{ - uint8_t version; /* set to 0 */ - uint8_t pad; - uint16_t len; - uint32_t present; - uint64_t MAC_timestamp; - uint8_t flags; -}; - -/* ++++++++++++ Wireless header (IEEE 802.11) ++++++++++++++++ */ -struct ndpi_wifi_header -{ - uint16_t fc; - uint16_t duration; - uint8_t rcvr[6]; - uint8_t trsm[6]; - uint8_t dest[6]; - uint16_t seq_ctrl; - /* uint64_t ccmp - for data encryption only - check fc.flag */ -}; - -/* +++++++++++++++++++++++ MPLS header +++++++++++++++++++++++ */ -struct ndpi_mpls_header -{ - /* Before using this strcut to parse an MPLS header, you will need to convert - * the 4-byte data to the correct endianess with ntohl(). */ - uint32_t ttl:8, s:1, exp:3, label:20; -}; - -extern union mpls { - uint32_t u32; - struct ndpi_mpls_header mpls; -} mpls; - -/* ++++++++++++++++++++++++ IP header ++++++++++++++++++++++++ */ -struct ndpi_iphdr { - uint8_t ihl:4, version:4; - uint8_t tos; - uint16_t tot_len; - uint16_t id; - uint16_t frag_off; - uint8_t ttl; - uint8_t protocol; - uint16_t check; - uint32_t saddr; - uint32_t daddr; -}; - -/* +++++++++++++++++++++++ IPv6 header +++++++++++++++++++++++ */ -/* rfc3542 */ -struct ndpi_in6_addr { - union { - uint8_t u6_addr8[16]; - uint16_t u6_addr16[8]; - uint32_t u6_addr32[4]; - uint64_t u6_addr64[2]; - } u6_addr; /* 128-bit IP6 address */ -}; - -struct ndpi_ip6_hdrctl { - uint32_t ip6_un1_flow; - uint16_t ip6_un1_plen; - uint8_t ip6_un1_nxt; - uint8_t ip6_un1_hlim; -}; - -struct ndpi_ipv6hdr { - struct ndpi_ip6_hdrctl ip6_hdr; - struct ndpi_in6_addr ip6_src; - struct ndpi_in6_addr ip6_dst; -}; - -/* +++++++++++++++++++++++ TCP header +++++++++++++++++++++++ */ -struct ndpi_tcphdr -{ - uint16_t source; - uint16_t dest; - uint32_t seq; - uint32_t ack_seq; - uint16_t res1:4, doff:4, fin:1, syn:1, rst:1, psh:1, ack:1, urg:1, ece:1, cwr:1; - uint16_t window; - uint16_t check; - uint16_t urg_ptr; -}; - -/* +++++++++++++++++++++++ UDP header +++++++++++++++++++++++ */ -struct ndpi_udphdr -{ - uint16_t source; - uint16_t dest; - uint16_t len; - uint16_t check; -}; -struct ndpi_dns_packet_header { - uint16_t tr_id; - uint16_t flags; - uint16_t num_queries; - uint16_t num_answers; - uint16_t authority_rrs; - uint16_t additional_rrs; -}; - -/* +++++++++++++++++++++++ ICMP header +++++++++++++++++++++++ */ -struct ndpi_icmphdr { - uint8_t type;/* message type */ - uint8_t code;/* type sub-code */ - uint16_t checksum; - union { - struct { - uint16_t id; - uint16_t sequence; - } echo; /* echo datagram */ - - uint32_t gateway; /* gateway address */ - struct { - uint16_t _unused; - uint16_t mtu; - } frag;/* path mtu discovery */ - } un; -}; - -/* +++++++++++++++++++++++ ICMP6 header +++++++++++++++++++++++ */ -struct ndpi_icmp6hdr { - uint8_t icmp6_type; /* type field */ - uint8_t icmp6_code; /* code field */ - uint16_t icmp6_cksum; /* checksum field */ - union { - uint32_t icmp6_un_data32[1]; /* type-specific field */ - uint16_t icmp6_un_data16[2]; /* type-specific field */ - uint8_t icmp6_un_data8[4]; /* type-specific field */ - } icmp6_dataun; -}; - -/* +++++++++++++++++++++++ VXLAN header +++++++++++++++++++++++ */ -struct ndpi_vxlanhdr { - uint16_t flags; - uint16_t groupPolicy; - uint32_t vni; -}; - -struct tinc_cache_entry { - uint32_t src_address; - uint32_t dst_address; - uint16_t dst_port; -}; -""" - -cc_ndpi_stuctures = """ - -#define NDPI_MAX_NUM_TLS_APPL_BLOCKS 8 - -typedef enum { - NDPI_LOG_ERROR, - NDPI_LOG_TRACE, - NDPI_LOG_DEBUG, - NDPI_LOG_DEBUG_EXTRA -} ndpi_log_level_t; - -typedef enum { - ndpi_l4_proto_unknown = 0, - ndpi_l4_proto_tcp_only, - ndpi_l4_proto_udp_only, - ndpi_l4_proto_tcp_and_udp, -} ndpi_l4_proto_info; - -typedef enum { - ndpi_no_tunnel = 0, - ndpi_gtp_tunnel, - ndpi_capwap_tunnel, - ndpi_tzsp_tunnel, - ndpi_l2tp_tunnel, -} ndpi_packet_tunnel; - -typedef enum { - NDPI_NO_RISK = 0, - NDPI_URL_POSSIBLE_XSS, - NDPI_URL_POSSIBLE_SQL_INJECTION, - NDPI_URL_POSSIBLE_RCE_INJECTION, - NDPI_BINARY_APPLICATION_TRANSFER, - NDPI_KNOWN_PROTOCOL_ON_NON_STANDARD_PORT, - NDPI_TLS_SELFSIGNED_CERTIFICATE, - NDPI_TLS_OBSOLETE_VERSION, - NDPI_TLS_WEAK_CIPHER, - NDPI_TLS_CERTIFICATE_EXPIRED, - NDPI_TLS_CERTIFICATE_MISMATCH, - NDPI_HTTP_SUSPICIOUS_USER_AGENT, - NDPI_HTTP_NUMERIC_IP_HOST, - NDPI_HTTP_SUSPICIOUS_URL, - NDPI_HTTP_SUSPICIOUS_HEADER, - NDPI_TLS_NOT_CARRYING_HTTPS, - NDPI_SUSPICIOUS_DGA_DOMAIN, - NDPI_MALFORMED_PACKET, - NDPI_SSH_OBSOLETE_CLIENT_VERSION_OR_CIPHER, - NDPI_SSH_OBSOLETE_SERVER_VERSION_OR_CIPHER, - NDPI_SMB_INSECURE_VERSION, - NDPI_TLS_SUSPICIOUS_ESNI_USAGE, - NDPI_UNSAFE_PROTOCOL, - NDPI_DNS_SUSPICIOUS_TRAFFIC, - NDPI_TLS_MISSING_SNI, - NDPI_HTTP_SUSPICIOUS_CONTENT, - NDPI_RISKY_ASN, - NDPI_RISKY_DOMAIN, - NDPI_MALICIOUS_JA3, - NDPI_MALICIOUS_SHA1_CERTIFICATE, - NDPI_DESKTOP_OR_FILE_SHARING_SESSION, - NDPI_TLS_UNCOMMON_ALPN, - NDPI_TLS_CERT_VALIDITY_TOO_LONG, - NDPI_TLS_SUSPICIOUS_EXTENSION, - NDPI_TLS_FATAL_ALERT, - NDPI_SUSPICIOUS_ENTROPY, - NDPI_CLEAR_TEXT_CREDENTIALS, - NDPI_DNS_LARGE_PACKET, - NDPI_DNS_FRAGMENTED, - NDPI_INVALID_CHARACTERS, - NDPI_POSSIBLE_EXPLOIT, - NDPI_TLS_CERTIFICATE_ABOUT_TO_EXPIRE, - NDPI_PUNYCODE_IDN, - NDPI_ERROR_CODE_DETECTED, - NDPI_HTTP_CRAWLER_BOT, - NDPI_ANONYMOUS_SUBSCRIBER, - - /* Leave this as last member */ - NDPI_MAX_RISK -} ndpi_risk_enum; - -typedef uint32_t ndpi_risk; - -/* NDPI_VISIT */ -typedef enum { - ndpi_preorder, - ndpi_postorder, - ndpi_endorder, - ndpi_leaf -} ndpi_VISIT; - -/* NDPI_NODE */ -typedef struct node_t { - char *key; - struct node_t *left, *right; -} ndpi_node; - -/* NDPI_MASK_SIZE */ -typedef uint32_t ndpi_ndpi_mask; - -/* NDPI_PROTO_BITMASK_STRUCT */ -typedef struct ndpi_protocol_bitmask_struct { - ndpi_ndpi_mask fds_bits[16]; -} NDPI_PROTOCOL_BITMASK; - -/* NDPI_PROTOCOL_BITTORRENT */ -typedef struct spinlock { - volatile int val; -} spinlock_t; - -typedef struct atomic { - volatile int counter; -} atomic_t; - -typedef long int time_t; - -struct hash_ip4p_node { - struct hash_ip4p_node *next, *prev; - time_t lchg; - uint16_t port,count:12,flag:4; - uint32_t ip; -}; - -struct hash_ip4p { - struct hash_ip4p_node *top; - spinlock_t lock; - size_t len; -}; - -struct hash_ip4p_table { - size_t size; - int ipv6; - spinlock_t lock; - atomic_t count; - struct hash_ip4p tbl; -}; - -struct bt_announce { // 192 bytes - uint32_t hash[5]; - uint32_t ip[4]; - uint32_t time; - uint16_t port; - uint8_t name_len, - name[149]; // 149 bytes -}; - -/* NDPI_PROTOCOL_TINC */ -#define TINC_CACHE_MAX_SIZE 10 - -typedef enum { - NDPI_HTTP_METHOD_UNKNOWN = 0, - NDPI_HTTP_METHOD_OPTIONS, - NDPI_HTTP_METHOD_GET, - NDPI_HTTP_METHOD_HEAD, - NDPI_HTTP_METHOD_PATCH, - NDPI_HTTP_METHOD_POST, - NDPI_HTTP_METHOD_PUT, - NDPI_HTTP_METHOD_DELETE, - NDPI_HTTP_METHOD_TRACE, - NDPI_HTTP_METHOD_CONNECT -} ndpi_http_method; - -struct ndpi_lru_cache_entry { - uint32_t key; /* Store the whole key to avoid ambiguities */ - uint32_t is_full:1, value:16, pad:15; -}; - -struct ndpi_lru_cache { - uint32_t num_entries; - struct ndpi_lru_cache_entry *entries; -}; - -typedef union -{ - uint32_t ipv4; - uint8_t ipv4_uint8_t[4]; - struct ndpi_in6_addr ipv6; -} ndpi_ip_addr_t; - -struct ndpi_id_struct { - /** - detected_protocol_bitmask: - access this bitmask to find out whether an id has used skype or not - if a flag is set here, it will not be reset - to compare this, use: - **/ - NDPI_PROTOCOL_BITMASK detected_protocol_bitmask; - - /* NDPI_PROTOCOL_IRC_MAXPORT % 2 must be 0 */ - /* NDPI_PROTOCOL_IRC */ -#define NDPI_PROTOCOL_IRC_MAXPORT 8 - uint16_t irc_port[NDPI_PROTOCOL_IRC_MAXPORT]; - uint32_t last_time_port_used[NDPI_PROTOCOL_IRC_MAXPORT]; - uint32_t irc_ts; - - /* NDPI_PROTOCOL_GNUTELLA */ - uint32_t gnutella_ts; - - /* NDPI_PROTOCOL_UNENCRYPTED_JABBER */ - uint32_t jabber_stun_or_ft_ts; - - /* NDPI_PROTOCOL_DIRECTCONNECT */ - uint32_t directconnect_last_safe_access_time; - - /* NDPI_PROTOCOL_DIRECTCONNECT */ - uint16_t detected_directconnect_port; - uint16_t detected_directconnect_udp_port; - uint16_t detected_directconnect_ssl_port; - - /* NDPI_PROTOCOL_UNENCRYPTED_JABBER */ -#define JABBER_MAX_STUN_PORTS 6 - uint16_t jabber_voice_stun_port[JABBER_MAX_STUN_PORTS]; - uint16_t jabber_file_transfer_port[2]; - - /* NDPI_PROTOCOL_GNUTELLA */ - uint16_t detected_gnutella_udp_port1; - uint16_t detected_gnutella_udp_port2; - - /* NDPI_PROTOCOL_IRC */ - uint8_t irc_number_of_port; - - /* NDPI_PROTOCOL_UNENCRYPTED_JABBER */ - uint8_t jabber_voice_stun_used_ports; -}; - -struct ndpi_flow_tcp_struct { - /* NDPI_PROTOCOL_MAIL_SMTP */ - uint16_t smtp_command_bitmask; - - /* NDPI_PROTOCOL_MAIL_POP */ - uint16_t pop_command_bitmask; - - /* NDPI_PROTOCOL_WHATSAPP */ - uint8_t wa_matched_so_far; - - /* NDPI_PROTOCOL_IRC */ - uint8_t irc_stage; - - /* NDPI_PROTOCOL_H323 */ - uint8_t h323_valid_packets; - - /* NDPI_PROTOCOL_GNUTELLA */ - uint8_t gnutella_msg_id[3]; - - /* NDPI_PROTOCOL_IRC */ - uint32_t irc_3a_counter:3; - uint32_t irc_stage2:5; - uint32_t irc_direction:2; - uint32_t irc_0x1000_full:1; - - /* NDPI_PROTOCOL_USENET */ - uint32_t usenet_stage:2; - - /* NDPI_PROTOCOL_HTTP */ - uint32_t http_stage:2; - uint32_t http_empty_line_seen:1; - - /* NDPI_PROTOCOL_GNUTELLA */ - uint32_t gnutella_stage:2; // 0 - 2 - - /* NDPI_PROTOCOL_SSH */ - uint32_t ssh_stage:3; - - /* NDPI_PROTOCOL_VNC */ - uint32_t vnc_stage:2; // 0 - 3 - - /* NDPI_PROTOCOL_TELNET */ - uint32_t telnet_stage:2; // 0 - 2 - - struct { - struct { - uint8_t *buffer; - unsigned buffer_len, buffer_used; - } message; - - /* NDPI_PROTOCOL_TLS */ - uint8_t certificate_processed:1, fingerprint_set:1, _pad:6; - uint8_t sha1_certificate_fingerprint[20], num_tls_blocks; - int16_t tls_application_blocks_len[NDPI_MAX_NUM_TLS_APPL_BLOCKS]; - } tls; - - /* NDPI_PROTOCOL_POSTGRES */ - uint32_t postgres_stage:3; - - /* Part of the TCP header. */ - uint32_t seen_syn:1; - uint32_t seen_syn_ack:1; - uint32_t seen_ack:1; - - /* NDPI_PROTOCOL_ICECAST */ - uint32_t icecast_stage:1; - - /* NDPI_PROTOCOL_DOFUS */ - uint32_t dofus_stage:1; - - /* NDPI_PROTOCOL_FIESTA */ - uint32_t fiesta_stage:2; - - /* NDPI_PROTOCOL_WORLDOFWARCRAFT */ - uint32_t wow_stage:2; - - /* NDPI_PROTOCOL_SHOUTCAST */ - uint32_t shoutcast_stage:2; - - /* NDPI_PROTOCOL_RTP */ - uint32_t rtp_special_packets_seen:1; - - /* NDPI_PROTOCOL_MAIL_POP */ - uint32_t mail_pop_stage:2; - - /* NDPI_PROTOCOL_MAIL_IMAP */ - uint32_t mail_imap_stage:3, mail_imap_starttls:2; - - /* NDPI_PROTOCOL_SKYPE */ - uint8_t skype_packet_id; - - /* NDPI_PROTOCOL_LOTUS_NOTES */ - uint8_t lotus_notes_packet_id; - - /* NDPI_PROTOCOL_TEAMVIEWER */ - uint8_t teamviewer_stage; - - /* NDPI_PROTOCOL_ZMQ */ - uint8_t prev_zmq_pkt_len; - uint8_t prev_zmq_pkt[10]; - - /* NDPI_PROTOCOL_PPSTREAM */ - uint32_t ppstream_stage:3; - - /* NDPI_PROTOCOL_MEMCACHED */ - uint8_t memcached_matches; - - /* NDPI_PROTOCOL_NEST_LOG_SINK */ - uint8_t nest_log_sink_matches; -}; - -struct ndpi_flow_udp_struct { - /* NDPI_PROTOCOL_PPSTREAM */ - uint32_t ppstream_stage:3; // 0 - 7 - - /* NDPI_PROTOCOL_HALFLIFE2 */ - uint32_t halflife2_stage:2; // 0 - 2 - - /* NDPI_PROTOCOL_TFTP */ - uint32_t tftp_stage:2; - - /* NDPI_PROTOCOL_AIMINI */ - uint32_t aimini_stage:5; - - /* NDPI_PROTOCOL_XBOX */ - uint32_t xbox_stage:1; - - /* NDPI_PROTOCOL_SKYPE */ - uint8_t skype_packet_id; - - /* NDPI_PROTOCOL_TEAMVIEWER */ - uint8_t teamviewer_stage; - - /* NDPI_PROTOCOL_EAQ */ - uint8_t eaq_pkt_id; - uint32_t eaq_sequence; - - /* NDPI_PROTOCOL_RX */ - uint32_t rx_conn_epoch; - uint32_t rx_conn_id; - - /* NDPI_PROTOCOL_MEMCACHED */ - uint8_t memcached_matches; - - /* NDPI_PROTOCOL_WIREGUARD */ - uint8_t wireguard_stage; - uint32_t wireguard_peer_index[2]; - - /* NDPI_PROTOCOL_QUIC */ - u_int8_t *quic_reasm_buf; - u_int32_t quic_reasm_buf_len; - - /* NDPI_PROTOCOL_CSGO */ - uint8_t csgo_strid[18],csgo_state,csgo_s2; - uint32_t csgo_id2; - - /* NDPI_PROTOCOL_RDP */ - u_int8_t rdp_to_srv[3], rdp_from_srv[3], rdp_to_srv_pkts, rdp_from_srv_pkts; - - /* NDPI_PROTOCOL_IMO */ - uint8_t imo_last_one_byte_pkt, imo_last_byte; - -}; - -struct ndpi_int_one_line_struct { - const uint8_t *ptr; - uint16_t len; -}; - -struct ndpi_packet_struct { - const struct ndpi_iphdr *iph; - const struct ndpi_ipv6hdr *iphv6; - const struct ndpi_tcphdr *tcp; - const struct ndpi_udphdr *udp; - const uint8_t *generic_l4_ptr; /* is set only for non tcp-udp traffic */ - const uint8_t *payload; - - uint64_t current_time_ms; - - uint16_t detected_protocol_stack[2]; - - struct ndpi_int_one_line_struct line[64]; - /* HTTP headers */ - struct ndpi_int_one_line_struct host_line; - struct ndpi_int_one_line_struct forwarded_line; - struct ndpi_int_one_line_struct referer_line; - struct ndpi_int_one_line_struct content_line; - struct ndpi_int_one_line_struct content_disposition_line; - struct ndpi_int_one_line_struct accept_line; - struct ndpi_int_one_line_struct user_agent_line; - struct ndpi_int_one_line_struct http_url_name; - struct ndpi_int_one_line_struct http_encoding; - struct ndpi_int_one_line_struct http_transfer_encoding; - struct ndpi_int_one_line_struct http_contentlen; - struct ndpi_int_one_line_struct http_cookie; - struct ndpi_int_one_line_struct http_origin; - struct ndpi_int_one_line_struct http_x_session_type; - struct ndpi_int_one_line_struct server_line; - struct ndpi_int_one_line_struct http_method; - struct ndpi_int_one_line_struct http_response; - uint8_t http_num_headers; /* number of found (valid) header lines in HTTP request or response */ - - uint16_t l3_packet_len; - uint16_t payload_packet_len; - uint16_t parsed_lines; - uint16_t empty_line_position; - uint8_t tcp_retransmission; - - uint8_t packet_lines_parsed_complete:1, - packet_direction:1, empty_line_position_set:1, http_check_content:1, pad:4; -}; - -struct ndpi_detection_module_struct; -struct ndpi_flow_struct; - -struct ndpi_call_function_struct { - NDPI_PROTOCOL_BITMASK detection_bitmask; - NDPI_PROTOCOL_BITMASK excluded_protocol_bitmask; - uint32_t ndpi_selection_bitmask; - void (*func) (struct ndpi_detection_module_struct *, struct ndpi_flow_struct *flow); - uint8_t detection_feature; -}; - -struct ndpi_subprotocol_conf_struct { - void (*func) (struct ndpi_detection_module_struct *, char *attr, char *value, int protocol_id); -}; - -typedef struct { - uint16_t port_low, port_high; -} ndpi_port_range; - -typedef enum { - NDPI_PROTOCOL_SAFE = 0, /* Surely doesn't provide risks for the network. (e.g., a news site) */ - NDPI_PROTOCOL_ACCEPTABLE, /* Probably doesn't provide risks, but could be malicious (e.g., Dropbox) */ - NDPI_PROTOCOL_FUN, /* Pure fun protocol, which may be prohibited by the user policy */ - NDPI_PROTOCOL_UNSAFE, /* Probably provides risks, but could be a normal traffic. Unencrypted protocols - with clear pass should be here (e.g., telnet) */ - NDPI_PROTOCOL_POTENTIALLY_DANGEROUS, /* Possibly dangerous (ex. Tor). */ - NDPI_PROTOCOL_DANGEROUS, /* Surely is dangerous (ex. smbv1). Be prepared to troubles */ - NDPI_PROTOCOL_TRACKER_ADS, /* Trackers, Advertisements... */ - NDPI_PROTOCOL_UNRATED /* No idea, not implemented or impossible to classify */ -} ndpi_protocol_breed_t; - -#define NUM_BREEDS 8 - -/* Abstract categories to group the protocols. */ -typedef enum { - NDPI_PROTOCOL_CATEGORY_UNSPECIFIED = 0, /* For general services and unknown protocols */ - NDPI_PROTOCOL_CATEGORY_MEDIA, /* Multimedia and streaming */ - NDPI_PROTOCOL_CATEGORY_VPN, /* Virtual Private Networks */ - NDPI_PROTOCOL_CATEGORY_MAIL, /* Protocols to send/receive/sync emails */ - NDPI_PROTOCOL_CATEGORY_DATA_TRANSFER, /* AFS/NFS and similar protocols */ - NDPI_PROTOCOL_CATEGORY_WEB, /* Web/mobile protocols and services */ - NDPI_PROTOCOL_CATEGORY_SOCIAL_NETWORK, /* Social networks */ - NDPI_PROTOCOL_CATEGORY_DOWNLOAD_FT, /* Download, FTP, file transfer/sharing */ - NDPI_PROTOCOL_CATEGORY_GAME, /* Online games */ - NDPI_PROTOCOL_CATEGORY_CHAT, /* Instant messaging */ - NDPI_PROTOCOL_CATEGORY_VOIP, /* Real-time communications and conferencing */ - NDPI_PROTOCOL_CATEGORY_DATABASE, /* Protocols for database communication */ - NDPI_PROTOCOL_CATEGORY_REMOTE_ACCESS, /* Remote access and control */ - NDPI_PROTOCOL_CATEGORY_CLOUD, /* Online cloud services */ - NDPI_PROTOCOL_CATEGORY_NETWORK, /* Network infrastructure protocols */ - NDPI_PROTOCOL_CATEGORY_COLLABORATIVE, /* Software for collaborative development, including Webmail */ - NDPI_PROTOCOL_CATEGORY_RPC, /* High level network communication protocols */ - NDPI_PROTOCOL_CATEGORY_STREAMING, /* Streaming protocols */ - NDPI_PROTOCOL_CATEGORY_SYSTEM_OS, /* System/Operating System level applications */ - NDPI_PROTOCOL_CATEGORY_SW_UPDATE, /* Software update */ - - /* See #define NUM_CUSTOM_CATEGORIES */ - NDPI_PROTOCOL_CATEGORY_CUSTOM_1, /* User custom category 1 */ - NDPI_PROTOCOL_CATEGORY_CUSTOM_2, /* User custom category 2 */ - NDPI_PROTOCOL_CATEGORY_CUSTOM_3, /* User custom category 3 */ - NDPI_PROTOCOL_CATEGORY_CUSTOM_4, /* User custom category 4 */ - NDPI_PROTOCOL_CATEGORY_CUSTOM_5, /* User custom category 5 */ - - /* Further categories... */ - NDPI_PROTOCOL_CATEGORY_MUSIC, - NDPI_PROTOCOL_CATEGORY_VIDEO, - NDPI_PROTOCOL_CATEGORY_SHOPPING, - NDPI_PROTOCOL_CATEGORY_PRODUCTIVITY, - NDPI_PROTOCOL_CATEGORY_FILE_SHARING, - - /* - The category below is used by sites who are used - to test connectivity - */ - NDPI_PROTOCOL_CATEGORY_CONNECTIVITY_CHECK, - - /* Some custom categories */ - CUSTOM_CATEGORY_MINING = 99, - CUSTOM_CATEGORY_MALWARE = 100, - CUSTOM_CATEGORY_ADVERTISEMENT = 101, - CUSTOM_CATEGORY_BANNED_SITE = 102, - CUSTOM_CATEGORY_SITE_UNAVAILABLE = 103, - CUSTOM_CATEGORY_ALLOWED_SITE = 104, - /* - The category below is used to track communications made by - security applications (e.g. sophosxl.net, spamhaus.org) - to track malware, spam etc. - */ - CUSTOM_CATEGORY_ANTIMALWARE = 105, - - /* - IMPORTANT - Please keep in sync with - static const char* categories[] = { ..} - in ndpi_main.c - */ - - NDPI_PROTOCOL_NUM_CATEGORIES - /* - NOTE: Keep this as last member - Unused as value but useful to getting the number of elements - in this datastructure - */ -} ndpi_protocol_category_t; - -typedef enum { - ndpi_pref_direction_detect_disable = 0, - ndpi_pref_enable_tls_block_dissection -} ndpi_detection_preference; - -/* ntop extensions */ -typedef struct ndpi_proto_defaults { - char *protoName; - ndpi_protocol_category_t protoCategory; - u_int16_t * subprotocols; - size_t subprotocol_count; - uint16_t protoId, protoIdx; - uint16_t tcp_default_ports[5], udp_default_ports[5]; - ndpi_protocol_breed_t protoBreed; - void (*func) (struct ndpi_detection_module_struct *, struct ndpi_flow_struct *flow); -} ndpi_proto_defaults_t; - -typedef struct ndpi_default_ports_tree_node { - ndpi_proto_defaults_t *proto; - uint8_t customUserProto; - uint16_t default_port; -} ndpi_default_ports_tree_node_t; - -typedef struct _ndpi_automa { - void *ac_automa; /* Real type is AC_AUTOMATA_t */ - uint8_t ac_automa_finalized; -} ndpi_automa; - -typedef struct ndpi_proto { - /* - Note - below we do not use ndpi_protocol_id_t as users can define their own - custom protocols and thus the typedef could be too short in size. - */ - uint16_t master_protocol /* e.g. HTTP */, app_protocol /* e.g. FaceBook */; - ndpi_protocol_category_t category; -} ndpi_protocol; - -#define NUM_CUSTOM_CATEGORIES 5 -#define CUSTOM_CATEGORY_LABEL_LEN 32 - - -struct ndpi_detection_module_struct { - NDPI_PROTOCOL_BITMASK detection_bitmask; - - uint32_t current_ts; - - uint16_t num_tls_blocks_to_follow; - - char custom_category_labels[NUM_CUSTOM_CATEGORIES][CUSTOM_CATEGORY_LABEL_LEN]; - /* callback function buffer */ - struct ndpi_call_function_struct callback_buffer[250]; - uint32_t callback_buffer_size; - - struct ndpi_call_function_struct callback_buffer_tcp_no_payload[250]; - uint32_t callback_buffer_size_tcp_no_payload; - - struct ndpi_call_function_struct callback_buffer_tcp_payload[250]; - uint32_t callback_buffer_size_tcp_payload; - - struct ndpi_call_function_struct callback_buffer_udp[250]; - uint32_t callback_buffer_size_udp; - - struct ndpi_call_function_struct callback_buffer_non_tcp_udp[250]; - uint32_t callback_buffer_size_non_tcp_udp; - - ndpi_default_ports_tree_node_t *tcpRoot, *udpRoot; - - ndpi_log_level_t ndpi_log_level; /* default error */ - - /* misc parameters */ - uint32_t tcp_max_retransmission_window_size; - - /* subprotocol registration handler */ - struct ndpi_subprotocol_conf_struct subprotocol_conf[250]; - - unsigned ndpi_num_supported_protocols; - unsigned ndpi_num_custom_protocols; - - /* HTTP/DNS/HTTPS host matching */ - ndpi_automa host_automa, /* Used for DNS/HTTPS */ - bigrams_automa, impossible_bigrams_automa; /* TOR */ - /* IMPORTANT: please update ndpi_finalize_initalization() whenever you add a new automa */ - - struct { - ndpi_automa hostnames, hostnames_shadow; - void *ipAddresses, *ipAddresses_shadow; /* Patricia */ - uint8_t categories_loaded; - } custom_categories; - - /* IP-based protocol detection */ - void *protocols_ptree; - - uint8_t ip_version_limit; - - /* NDPI_PROTOCOL_OOKLA */ - struct ndpi_lru_cache *ookla_cache; - - /* NDPI_PROTOCOL_TINC */ - struct cache *tinc_cache; - - /* NDPI_PROTOCOL_STUN and subprotocols */ - struct ndpi_lru_cache *stun_cache; - - /* NDPI_PROTOCOL_MSTEAMS */ - struct ndpi_lru_cache *msteams_cache; - - ndpi_proto_defaults_t proto_defaults[512]; - - uint8_t direction_detect_disable:1, /* disable internal detection of packet direction */ - _pad:7; - - /* Current packet */ - struct ndpi_packet_struct packet; -}; - -#define NDPI_CIPHER_SAFE 0 -#define NDPI_CIPHER_WEAK 1 -#define NDPI_CIPHER_INSECURE 2 - -typedef enum { - ndpi_cipher_safe = NDPI_CIPHER_SAFE, - ndpi_cipher_weak = NDPI_CIPHER_WEAK, - ndpi_cipher_insecure = NDPI_CIPHER_INSECURE -} ndpi_cipher_weakness; - -struct ndpi_flow_struct { - uint16_t detected_protocol_stack[2]; - /* init parameter, internal used to set up timestamp,... */ - uint16_t guessed_protocol_id, guessed_host_protocol_id, guessed_category, guessed_header_category; - uint8_t l4_proto, protocol_id_already_guessed:1, host_already_guessed:1, - init_finished:1, setup_packet_direction:1, packet_direction:1, check_extra_packets:1; - /* - if ndpi_struct->direction_detect_disable == 1 - tcp sequence number connection tracking - */ - uint32_t next_tcp_seq_nr[2]; - uint8_t max_extra_packets_to_check; - uint8_t num_extra_packets_checked; - uint8_t num_processed_pkts; /* <= WARNING it can wrap but we do expect people to giveup earlier */ - - int (*extra_packets_func) (struct ndpi_detection_module_struct *, struct ndpi_flow_struct *flow); - /* - the tcp / udp / other l4 value union - used to reduce the number of bytes for tcp or udp protocol states - */ - union { - struct ndpi_flow_tcp_struct tcp; - struct ndpi_flow_udp_struct udp; - } l4; - - /* Place textual flow info here */ - char flow_extra_info[16]; - - char host_server_name[80]; - - uint8_t initial_binary_bytes[8], initial_binary_bytes_len; - uint8_t risk_checked; - ndpi_risk risk; /* Issues found with this flow [bitmask of ndpi_risk] */ - - /* - This structure below will not stay inside the protos - structure below as HTTP is used by many subprotocols - such as Facebook, Google... so it is hard to know - when to use it or not. Thus we leave it outside for the - time being. - */ - struct { - ndpi_http_method method; - uint8_t request_version; /* 0=1.0 and 1=1.1. Create an enum for this? */ - uint16_t response_status_code; /* 200, 404, etc. */ - char *url, *content_type, *user_agent; - char *detected_os; /* Via HTTP/QUIC User-Agent */ - char *nat_ip; /* Via HTTP X-Forwarded-For */ - - } http; - - /* - Put outside of the union to avoid issues in case the protocol - is remapped to somethign pther than Kerberos due to a faulty - dissector - */ - struct { - char *pktbuf; - uint16_t pktbuf_maxlen, pktbuf_currlen; - } kerberos_buf; - - struct { - u_int8_t num_udp_pkts, num_binding_requests; - u_int16_t num_processed_pkts; - } stun; - - /* TODO: something clever to save memory */ - struct { - uint8_t auth_found:1, auth_failed:1, auth_tls:1, auth_done:1, _pad:4; - char username[32], password[16]; - } ftp_imap_pop_smtp; - - union { - /* the only fields useful for nDPI and ntopng */ - struct { - uint8_t num_queries, num_answers, reply_code, is_query; - uint16_t query_type, query_class, rsp_type; - ndpi_ip_addr_t rsp_addr; /* The first address in a DNS response packet */ - } dns; - - struct { - uint8_t request_code; - uint8_t version; - } ntp; - - struct { - char hostname[48], domain[48], username[48]; - } kerberos; - - struct { - char *server_names, *alpn, *tls_supported_versions, *issuerDN, *subjectDN; - uint32_t notBefore, notAfter; - char ja3_client[33], ja3_server[33]; - uint16_t server_cipher; - u_int8_t sha1_certificate_fingerprint[20]; - u_int8_t hello_processed:1, subprotocol_detected:1, _pad:6; - struct { - uint16_t cipher_suite; - char *esni; - } encrypted_sni; - ndpi_cipher_weakness server_unsafe_cipher; - uint16_t ssl_version, server_names_len; - } tls_quic; - - struct { - char client_signature[48], server_signature[48]; - char hassh_client[33], hassh_server[33]; - } ssh; - - struct { - uint8_t username_detected:1, username_found:1, - password_detected:1, password_found:1, - pad:4; - uint8_t character_id; - char username[32], password[32]; - } telnet; - - struct { - char answer[96]; - } mdns; - - struct { - char version[32]; - } ubntac2; - - struct { - /* Bittorrent hash */ - uint8_t hash[20]; - } bittorrent; - - struct { - char fingerprint[48]; - char class_ident[48]; - } dhcp; - } protos; - - /*** ALL protocol specific 64 bit variables here ***/ - - /* protocols which have marked a connection as this connection cannot be protocol XXX, multiple uint64_t */ - NDPI_PROTOCOL_BITMASK excluded_protocol_bitmask; - - ndpi_protocol_category_t category; - - /* NDPI_PROTOCOL_REDIS */ - uint8_t redis_s2d_first_char, redis_d2s_first_char; - - uint16_t packet_counter; // can be 0 - 65000 - uint16_t packet_direction_counter[2]; - uint16_t byte_counter[2]; - /* NDPI_PROTOCOL_BITTORRENT */ - uint8_t bittorrent_stage; // can be 0 - 255 - - /* NDPI_PROTOCOL_DIRECTCONNECT */ - uint8_t directconnect_stage:2; // 0 - 1 - - /* NDPI_PROTOCOL_HTTP */ - uint8_t http_detected:1; - - /* NDPI_PROTOCOL_RTSP */ - uint8_t rtsprdt_stage:2; - - /* NDPI_PROTOCOL_ZATTOO */ - uint8_t zattoo_stage:3; - - /* NDPI_PROTOCOL_THUNDER */ - uint8_t thunder_stage:2; // 0 - 3 - - /* NDPI_PROTOCOL_FLORENSIA */ - uint8_t florensia_stage:1; - - /* NDPI_PROTOCOL_SOCKS */ - uint8_t socks5_stage:2, socks4_stage:2; // 0 - 3 - - /* NDPI_PROTOCOL_EDONKEY */ - uint8_t edonkey_stage:2; // 0 - 3 - - /* NDPI_PROTOCOL_FTP_CONTROL */ - uint8_t ftp_control_stage:2; - - /* NDPI_PROTOCOL_RTMP */ - uint8_t rtmp_stage:2; - - /* NDPI_PROTOCOL_PANDO */ - uint8_t pando_stage:3; - - /* NDPI_PROTOCOL_STEAM */ - uint16_t steam_stage:3, steam_stage1:3, steam_stage2:2, steam_stage3:2; - - /* NDPI_PROTOCOL_PPLIVE */ - uint8_t pplive_stage1:3, pplive_stage2:2, pplive_stage3:2; - - /* NDPI_PROTOCOL_STARCRAFT */ - uint8_t starcraft_udp_stage : 3; // 0-7 - - /* NDPI_PROTOCOL_OPENVPN */ - uint8_t ovpn_session_id[8]; - uint8_t ovpn_counter; - - /* Flow key used to search a match into the mining cache */ - u_int32_t key_mining_cache; - - /* NDPI_PROTOCOL_TINC */ - uint8_t tinc_state; - struct tinc_cache_entry tinc_cache_entry; - - struct ndpi_id_struct *src; - struct ndpi_id_struct *dst; -}; - -typedef struct { - char *string_to_match, *proto_name; - int protocol_id; - ndpi_protocol_category_t protocol_category; - ndpi_protocol_breed_t protocol_breed; -} ndpi_protocol_match; - -typedef struct { - char *string_to_match; - ndpi_protocol_category_t protocol_category; -} ndpi_category_match; - -typedef struct { - uint32_t network; - uint8_t cidr; - uint8_t value; -} ndpi_network; - -typedef uint32_t ndpi_init_prefs; - -typedef enum { - ndpi_no_prefs = 0, - ndpi_dont_load_tor_hosts, -} ndpi_prefs; - -typedef struct { - int protocol_id; - ndpi_protocol_category_t protocol_category; - ndpi_protocol_breed_t protocol_breed; -} ndpi_protocol_match_result; - -typedef struct { - char *str; - uint16_t str_len; -} ndpi_string; - -/* **************************************** */ - -struct ndpi_analyze_struct { - uint32_t *values; - uint32_t min_val, max_val, sum_total, num_data_entries, next_value_insert_index; - uint16_t num_values_array_len /* lenght of the values array */; - - struct { - float mu, q; - } stddev; -}; - -#define DEFAULT_SERIES_LEN 64 -#define MAX_SERIES_LEN 512 -#define MIN_SERIES_LEN 8 - -typedef struct ndpi_ptree ndpi_ptree_t; - -""" - -cc_ndpi_apis = """ -struct ndpi_detection_module_struct *ndpi_init_detection_module(void); -void *memset(void *str, int c, size_t n); -void ndpi_set_protocol_detection_bitmask2(struct ndpi_detection_module_struct *ndpi_struct, - const NDPI_PROTOCOL_BITMASK * detection_bitmask); -ndpi_protocol ndpi_detection_process_packet(struct ndpi_detection_module_struct *ndpi_struct, - struct ndpi_flow_struct *flow, - const unsigned char *packet, - const unsigned short packetlen, - const uint64_t current_tick, - struct ndpi_id_struct *src, - struct ndpi_id_struct *dst); -ndpi_protocol ndpi_detection_giveup(struct ndpi_detection_module_struct *ndpi_struct, - struct ndpi_flow_struct *flow, - uint8_t enable_guess, - uint8_t *protocol_was_guessed); - -void * ndpi_malloc(size_t size); -void ndpi_free(void *ptr); -void * ndpi_flow_malloc(size_t size); -void ndpi_flow_free(void *ptr); -void ndpi_exit_detection_module(struct ndpi_detection_module_struct *ndpi_struct); -char* ndpi_protocol2name(struct ndpi_detection_module_struct *ndpi_mod, - ndpi_protocol proto, - char *buf, unsigned buf_len); -const char* ndpi_category_get_name(struct ndpi_detection_module_struct *ndpi_mod, ndpi_protocol_category_t category); -char* ndpi_revision(void); -void ndpi_finalize_initalization(struct ndpi_detection_module_struct *ndpi_str); -uint32_t ndpi_detection_get_sizeof_ndpi_flow_struct(void); -uint32_t ndpi_detection_get_sizeof_ndpi_id_struct(void); -uint32_t ndpi_detection_get_sizeof_ndpi_flow_tcp_struct(void); -uint32_t ndpi_detection_get_sizeof_ndpi_flow_udp_struct(void); -uint8_t ndpi_extra_dissection_possible(struct ndpi_detection_module_struct *ndpi_struct, struct ndpi_flow_struct *flow); -""" - - -def check_structures_size(flow_struct_defined, flow_struct_loaded, - id_struct_defined, id_struct_loaded, - tcp_flow_struct_defined, tcp_flow_struct_loaded, - udp_flow_struct_defined, udp_flow_struct_loaded): - """ Function used to check loaded structures sizes againt defined ones """ - errors = [] - if flow_struct_defined != flow_struct_loaded: - errors.append('ndpi_flow_struct') - if id_struct_defined != id_struct_loaded: - errors.append('ndpi_id_struct') - if tcp_flow_struct_defined != tcp_flow_struct_loaded: - errors.append('ndpi_tcp_flow_struct') - if udp_flow_struct_defined != udp_flow_struct_loaded: - errors.append('ndpi_udp_flow_struct') - return errors - - -class NDPI(): - """ ndpi module main class """ - - def __init__(self, libpath=None, max_tcp_dissections=80, max_udp_dissections=16, enable_guess=True): - self._ffi = cffi.FFI() - if libpath is None: - self._ndpi = self._ffi.dlopen(dirname(abspath(__file__)) + '/libndpi.so') - else: - self._ndpi = self._ffi.dlopen(libpath) - self._ffi.cdef(cc_ndpi_network_headers, packed=True) - self._ffi.cdef(cc_ndpi_stuctures) - self._ffi.cdef(cc_ndpi_apis) - self._mod = self._ndpi.ndpi_init_detection_module() - ndpi_revision = self._ffi.string(self._ndpi.ndpi_revision()).decode('utf-8', errors='ignore') - if ndpi_revision[:3] >= '3.1': - self._ndpi.ndpi_finalize_initalization(self._mod) - all = self._ffi.new('NDPI_PROTOCOL_BITMASK*') - self._ndpi.memset(self._ffi.cast("char *", all), 0xFF, self._ffi.sizeof("NDPI_PROTOCOL_BITMASK")) - self._ndpi.ndpi_set_protocol_detection_bitmask2(self._mod, all) - errors = check_structures_size(self._ffi.sizeof("struct ndpi_flow_struct"), - self._ndpi.ndpi_detection_get_sizeof_ndpi_flow_struct(), - self._ffi.sizeof("struct ndpi_id_struct"), - self._ndpi.ndpi_detection_get_sizeof_ndpi_id_struct(), - self._ffi.sizeof("struct ndpi_flow_tcp_struct"), - self._ndpi.ndpi_detection_get_sizeof_ndpi_flow_tcp_struct(), - self._ffi.sizeof("struct ndpi_flow_udp_struct"), - self._ndpi.ndpi_detection_get_sizeof_ndpi_flow_udp_struct()) - if len(errors) != 0: - raise ValueError('nDPI error: mismatch in the headers of following structures{}'.format(', '.join(errors))) - else: - self.SIZEOF_FLOW_STRUCT = self._ffi.sizeof("struct ndpi_flow_struct") - self.SIZEOF_ID_STRUCT = self._ffi.sizeof("struct ndpi_id_struct") - self.NULL = self._ffi.NULL - self.max_tcp_dissections = max_tcp_dissections - self.max_udp_dissections = max_udp_dissections - self.enable_guess = enable_guess - - def new_ndpi_flow(self): - """ Create a new nDPI flow object """ - f = self._ffi.cast('struct ndpi_flow_struct*', self._ndpi.ndpi_flow_malloc(self.SIZEOF_FLOW_STRUCT)) - self._ndpi.memset(f, 0, self.SIZEOF_FLOW_STRUCT) - return f - - def new_ndpi_id(self): - """ Create a new nDPI id object """ - i = self._ffi.cast('struct ndpi_id_struct*', self._ndpi.ndpi_malloc(self.SIZEOF_ID_STRUCT)) - self._ndpi.memset(i, 0, self.SIZEOF_ID_STRUCT) - return i - - def ndpi_detection_process_packet(self, flow, packet, packetlen, current_tick, src, dst): - """ Main detection processing function """ - p = self._ndpi.ndpi_detection_process_packet(self._mod, flow, packet, packetlen, current_tick, src, dst) - return p - - def ndpi_detection_giveup(self, flow): - """ Giveup detection function """ - return self._ndpi.ndpi_detection_giveup(self._mod, flow, self.enable_guess, self._ffi.new("uint8_t*", 0)) - - def ndpi_flow_free(self, flow): - """ Free nDPI flow object """ - return self._ndpi.ndpi_flow_free(flow) - - def ndpi_free(self, ptr): - """ Free nDPI object """ - return self._ndpi.ndpi_free(ptr) - - def get_str_field(self, ptr): - """ Get fixed string size attribute """ - if ptr == self._ffi.NULL: - return '' - else: - return self._ffi.string(ptr).decode('utf-8', errors='ignore') - - def get_buffer_field(self, ptr, li): - """ Get variable string size attribute """ - if ptr == self._ffi.NULL: - return '' - else: - return self._ffi.string(ptr, li).decode('utf-8', errors='ignore') - - def ndpi_protocol2name(self, proto): - """ Convert nDPI protocol object to readable name """ - buf = self._ffi.new("char[32]") - self._ndpi.ndpi_protocol2name(self._mod, proto, buf, self._ffi.sizeof(buf)) - return self._ffi.string(buf).decode('utf-8', errors='ignore') - - def ndpi_category_get_name(self, category): - """ Convert nDPI protocol object to readable name """ - return self._ffi.string(self._ndpi.ndpi_category_get_name(self._mod, category)).decode('utf-8', errors='ignore') - - def ndpi_extra_dissection_possible(self, flow): - return self._ndpi.ndpi_extra_dissection_possible(self._mod, flow) - - def ndpi_exit_detection_module(self): - """ Exit function for nDPI module """ - self._ndpi.ndpi_exit_detection_module(self._mod) - self._ffi.dlclose(self._ndpi) diff --git a/python/ndpi/__init__.py b/python/ndpi/__init__.py new file mode 100644 index 000000000..f6f5b9079 --- /dev/null +++ b/python/ndpi/__init__.py @@ -0,0 +1,21 @@ +""" +------------------------------------------------------------------------------------------------------------------------ +__init__.py +Copyright (C) 2011-22 - ntop.org +This file is part of nDPI, an open source deep packet inspection library. +nDPI is free software: you can redistribute it and/or modify it under the terms of the GNU Lesser General Public +License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later +version. +nDPI is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details. +You should have received a copy of the GNU Lesser General Public License along with NFStream. +If not, see <http://www.gnu.org/licenses/>. +------------------------------------------------------------------------------------------------------------------------ +""" + +from .ndpi import NDPI, NDPIFlow + + +__author__ = """Zied Aouini""" +__email__ = 'aouinizied@gmail.com' +__version__ = '4.3.0' diff --git a/python/ndpi/ndpi.py b/python/ndpi/ndpi.py new file mode 100644 index 000000000..0dd5b56cf --- /dev/null +++ b/python/ndpi/ndpi.py @@ -0,0 +1,101 @@ +""" +------------------------------------------------------------------------------------------------------------------------ +ndpi.py +Copyright (C) 2011-22 - ntop.org +This file is part of nDPI, an open source deep packet inspection library. +nDPI is free software: you can redistribute it and/or modify it under the terms of the GNU Lesser General Public +License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later +version. +nDPI is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details. +You should have received a copy of the GNU Lesser General Public License along with NFStream. +If not, see <http://www.gnu.org/licenses/>. +------------------------------------------------------------------------------------------------------------------------ +""" + +from collections import namedtuple +from _ndpi import ffi, lib + + +ndpi_protocol = namedtuple('NDPIProtocol', ['C', + 'master_protocol', + 'app_protocol', + 'category']) + +ndpi_confidence = namedtuple('NDPIConfidence', ['id', + 'name']) + + +class NDPI(object): + __slots__ = ("_api_version", + "_revision", + "_detection_module") + + def __init__(self): + self._detection_module = lib.ndpi_init_detection_module(0) + if self._detection_module == ffi.NULL: + raise MemoryError("Unable to instantiate NDPI object") + lib.ndpi_py_setup_detection_module(self._detection_module) + + @property + def api_version(self): + return lib.ndpi_get_api_version() + + @property + def revision(self): + return ffi.string(lib.ndpi_revision()).decode('utf-8', errors='ignore') + + def process_packet(self, flow, packet, packet_time_ms): + p = lib.ndpi_detection_process_packet(self._detection_module, + flow._C, + packet, + len(packet), + int(packet_time_ms)) + return ndpi_protocol(C=p, + master_protocol=p.master_protocol, + app_protocol=p.app_protocol, + category=p.category) + + def giveup(self, flow, enable_guess=True): + p = lib.ndpi_detection_giveup(self._detection_module, + flow._C, + enable_guess, + ffi.new("uint8_t*", 0)) + return ndpi_protocol(C=p, + master_protocol=p.master_protocol, + app_protocol=p.app_protocol, + category=p.category) + + def protocol_name(self, protocol): + buf = ffi.new("char[40]") + lib.ndpi_protocol2name(self._detection_module, protocol.C, buf, ffi.sizeof(buf)) + return ffi.string(buf).decode('utf-8', errors='ignore') + + def protocol_category_name(self, protocol): + return ffi.string(lib.ndpi_category_get_name(self._detection_module, + protocol.C.category)).decode('utf-8', + errors='ignore') + + def __del__(self): + if self._detection_module != ffi.NULL: + lib.ndpi_exit_detection_module(self._detection_module) + + +class NDPIFlow(object): + __slots__ = ("_C") + + @property + def confidence(self): + confidence = self._C.confidence + return ndpi_confidence(id=confidence, + name=ffi.string(lib.ndpi_confidence_get_name(confidence)).decode('utf-8', + errors='ignore')) + + def __init__(self): + self._C = lib.ndpi_py_initialize_flow() + + def __del__(self): + if self._C != ffi.NULL: + lib.ndpi_flow_free(self._C) + self._C = ffi.NULL + diff --git a/python/ndpi/ndpi_build.py b/python/ndpi/ndpi_build.py new file mode 100644 index 000000000..8bc412126 --- /dev/null +++ b/python/ndpi/ndpi_build.py @@ -0,0 +1,137 @@ +""" +------------------------------------------------------------------------------------------------------------------------ +ndpi_build.py +Copyright (C) 2011-22 - ntop.org +This file is part of nDPI, an open source deep packet inspection library. +nDPI is free software: you can redistribute it and/or modify it under the terms of the GNU Lesser General Public +License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later +version. +nDPI is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details. +You should have received a copy of the GNU Lesser General Public License along with NFStream. +If not, see <http://www.gnu.org/licenses/>. +------------------------------------------------------------------------------------------------------------------------ +""" + +from cffi import FFI +import subprocess +import pathlib + +NDPI_INCLUDES = """ +#include "ndpi_main.h" +#include "ndpi_typedefs.h" +#include "ndpi_api.h" +""" + +NDPI_HELPERS = """ +// nDPI cffi helper functions (function naming convention ndpi_py_*) + +void ndpi_py_setup_detection_module(struct ndpi_detection_module_struct *mod) { + if (mod == NULL) { + return; + } else { + NDPI_PROTOCOL_BITMASK protos; + NDPI_BITMASK_SET_ALL(protos); // Set bitmask for ALL protocols + ndpi_set_protocol_detection_bitmask2(mod, &protos); + ndpi_finalize_initialization(mod); + } +}; + +struct ndpi_flow_struct * ndpi_py_initialize_flow(void) { + struct ndpi_flow_struct * ndpi_flow = NULL; + ndpi_flow = (struct ndpi_flow_struct *)ndpi_flow_malloc(SIZEOF_FLOW_STRUCT); + memset(ndpi_flow, 0, SIZEOF_FLOW_STRUCT); + return ndpi_flow; +}; +""" + + +NDPI_APIS = """ +u_int16_t ndpi_get_api_version(void); +char* ndpi_revision(void); +struct ndpi_detection_module_struct *ndpi_init_detection_module(ndpi_init_prefs prefs); +void ndpi_exit_detection_module(struct ndpi_detection_module_struct *ndpi_struct); +void ndpi_flow_free(void *ptr); +ndpi_protocol ndpi_detection_process_packet(struct ndpi_detection_module_struct *ndpi_struct, + struct ndpi_flow_struct *flow, + const unsigned char *packet, + const unsigned short packetlen, + const u_int64_t packet_time_ms); +ndpi_protocol ndpi_detection_giveup(struct ndpi_detection_module_struct *ndpi_struct, + struct ndpi_flow_struct *flow, + u_int8_t enable_guess, + u_int8_t *protocol_was_guessed); +void ndpi_py_setup_detection_module(struct ndpi_detection_module_struct *mod); +struct ndpi_flow_struct * ndpi_py_initialize_flow(void); +char* ndpi_protocol2name(struct ndpi_detection_module_struct *ndpi_mod, ndpi_protocol proto, char *buf, u_int buf_len); +const char* ndpi_category_get_name(struct ndpi_detection_module_struct *ndpi_mod, ndpi_protocol_category_t category); +const char* ndpi_confidence_get_name(ndpi_confidence_t confidence); +""" + +ffi_builder = FFI() + + +INCLUDE_DIR = pathlib.Path(__file__)\ + .parent.resolve().parent.resolve().parent.resolve().\ + joinpath("src").joinpath("include") + +LIBRARY_DIR = pathlib.Path(__file__)\ + .parent.resolve().parent.resolve().parent.resolve().\ + joinpath("src").joinpath("lib") + + +NDPI_CDEF = subprocess.run(["gcc", + "-DNDPI_LIB_COMPILATION", + "-DNDPI_CFFI_PREPROCESSING", + "-DNDPI_CFFI_PREPROCESSING_EXCLUDE_PACKED", + "-E", "-x", "c", "-P", "-C", + str(INCLUDE_DIR.joinpath("ndpi_typedefs.h"))], + capture_output=True + ).stdout.decode('utf-8', + errors='ignore') + +NDPI_MODULE_STRUCT_CDEF = NDPI_CDEF.split("//CFFI.NDPI_MODULE_STRUCT")[1] + + +NDPI_PACKED = subprocess.run(["gcc", + "-DNDPI_LIB_COMPILATION", "-DNDPI_CFFI_PREPROCESSING", + "-E", "-x", "c", "-P", "-C", + str(INCLUDE_DIR.joinpath("ndpi_typedefs.h"))], + capture_output=True + ).stdout.decode('utf-8', + errors='ignore') + +NDPI_PACKED_STRUCTURES = NDPI_PACKED.split("//CFFI.NDPI_PACKED_STRUCTURES")[1] + +NDPI_SOURCE = NDPI_INCLUDES + NDPI_MODULE_STRUCT_CDEF + NDPI_HELPERS + + +ffi_builder.set_source("_ndpi", + NDPI_SOURCE, + libraries=["ndpi"], + library_dirs=[str(LIBRARY_DIR)], + include_dirs=[str(INCLUDE_DIR)]) + + +ffi_builder.cdef(""" +typedef uint64_t u_int64_t; +typedef uint32_t u_int32_t; +typedef uint16_t u_int16_t; +typedef uint8_t u_int8_t; +typedef uint8_t u_char; +typedef unsigned u_int; +struct in_addr { + unsigned long s_addr; +}; +struct in6_addr { + unsigned char s6_addr[16]; +}; +""") + +ffi_builder.cdef(NDPI_PACKED_STRUCTURES, packed=True) +ffi_builder.cdef(NDPI_CDEF) +ffi_builder.cdef(NDPI_APIS) + + +if __name__ == "__main__": + ffi_builder.compile(verbose=True) diff --git a/python/ndpi_example.py b/python/ndpi_example.py index d134d3947..8606ae84b 100755..100644 --- a/python/ndpi_example.py +++ b/python/ndpi_example.py @@ -1,273 +1,167 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- - """ -file: ndpi_example.py -This file is part of nDPI. - -Copyright (C) 2011-19 - ntop.org -Copyright (C) 2019 - Zied Aouini <aouinizied@gmail.com> (Incremental improvements) - -nDPI is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License -as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. - +------------------------------------------------------------------------------------------------------------------------ +ndpi_example.py +Copyright (C) 2011-22 - ntop.org +This file is part of nDPI, an open source deep packet inspection library. +nDPI is free software: you can redistribute it and/or modify it under the terms of the GNU Lesser General Public +License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later +version. nDPI is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty -of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. - -You should have received a copy of the GNU General Public License along with nDPI. +of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details. +You should have received a copy of the GNU Lesser General Public License along with NFStream. If not, see <http://www.gnu.org/licenses/>. +------------------------------------------------------------------------------------------------------------------------ """ -from ndpi_typestruct import * -from ctypes import * -from scapy.all import * -import sys - -# ------- return type & pcapstruct to declare ------- - - -class WorkFlow(Structure): - _fields_ = [("src_ip", c_uint32), - ("dst_ip", c_uint32), - ("src_port", c_uint16), - ("dst_port", c_uint16), - ("protocol", c_uint8), - ("packets", c_uint32), - ("detected_protocol", NDPIProtocol), - ("detection_completed", c_uint8), - ("id", c_uint32), - ("src_id", POINTER(NDPIIdStruct)), - ("dst_id", POINTER(NDPIIdStruct)), - ("flow", POINTER(NDPIFlowStruct))] - - -CMCFUN = CFUNCTYPE(c_int, c_void_p, c_void_p) -GUESS = CFUNCTYPE(None, c_void_p, c_int32, c_int, c_void_p) -FREE = CFUNCTYPE(None, c_void_p) - - -def node_proto_guess_walker(nodo, which, depth, user_data): - global ndpi_info_mod +from collections import namedtuple +from ndpi import NDPI, NDPIFlow +import argparse +import socket +import dpkt - flow = cast(nodo, POINTER(POINTER(WorkFlow))).contents.contents - if which == 0 or which == 3: # execute only preorder operation of the tree - if flow.detection_completed == 0: # order for tree operation - flow.detected_protocol = ndpi.ndpi_detection_giveup(ndpi_info_mod, - flow.flow, - 1, - cast(addressof(c_uint8(0)), POINTER(c_uint8))) - count_protocol[flow.detected_protocol.app_protocol] += flow.packets +FLOW_KEY = "{} {}:{} <-> {}:{}" +FLOW_STR = " {} {} [protocol:{}] [category:{}] [confidence:{}] [{} packets/{} bytes]" -def py_cmp_fun(a, b): - fa = cast(a, POINTER(WorkFlow)) - fb = cast(b, POINTER(WorkFlow)) - if fa.contents.src_ip < fb.contents.src_ip: return -1 - elif fa.contents.src_ip > fb.contents.src_ip: return 1 - if fa.contents.src_port < fb.contents.src_port: return -1 - elif fa.contents.src_port > fb.contents.src_port: return 1 - if fa.contents.dst_ip < fb.contents.dst_ip: return -1 - elif fa.contents.dst_ip > fb.contents.dst_ip: return 1 - if fa.contents.dst_port < fb.contents.dst_port: return -1 - elif fa.contents.dst_port > fb.contents.dst_port: return 1 - if fa.contents.protocol < fb.contents.protocol: return -1 - elif fa.contents.protocol > fb.contents.protocol: return 1 - return 0 +PROTOCOL_UNKNWON = 0 -def freer(a): - pass +class Flow(object): + __slots__ = ("index", + "pkts", + "bytes", + "detected_protocol", + "ndpi_flow") + def __init__(self): + self.pkts = 0 + self.detected_protocol = None + self.bytes = 0 + self.ndpi_flow = None -cmp_func = CMCFUN(py_cmp_fun) -guess_walker = GUESS(node_proto_guess_walker) -free_walk = FREE(freer) -# -------------------------------------- - -# number of analyzed packet -packet_number = 0 -flow_count = 0 -max_num_udp_dissected_pkts = 16 -max_num_tcp_dissected_pkts = 10 -flows_root = c_void_p(None) -flows_root_ref = pointer(flows_root) -count_protocol = (c_int32 * (ndpi.ndpi_wrap_ndpi_max_supported_protocols() + ndpi.ndpi_wrap_ndpi_max_num_custom_protocols() + 1))() -lista = [] # used to avoid impropriate memory deallocation from python - - -# check ndpi version -if ndpi.ndpi_get_api_version() != ndpi.ndpi_wrap_get_api_version(): - print("nDPI Library version mismatch: please make sure this code and the nDPI library are in sync\n") - sys.exit(-1) - -# create data structure of ndpi -ndpi_info_mod = ndpi.ndpi_init_detection_module() -if ndpi_info_mod is None: - sys.exit(-1) -else: - ndpi_ndpi_finalize_initalization(ndpi_info_mod) - - -def ip2int(ip): - """ - Convert an IP string to long and then c_uint32 - """ - packedIP = socket.inet_aton(ip) - return int(struct.unpack("!I", packedIP)[0]) - - -def get_flow(pkt): - global flows_root - global flows_root_ref - global flow_count - - ip_packet = pkt[1] - ip_protocol = ip_packet.proto - transport_packet = None - - if ip_protocol == 6 or ip_protocol == 17: - transport_packet = pkt[2] - if transport_packet is not None: - # to avoid two nodes in one binary tree for a flow - ip_src = ip2int(ip_packet.src) - ip_dst = ip2int(ip_packet.dst) - src_port = transport_packet.sport - dst_port = transport_packet.dport - else: - return None - # set flows to correct type and data - ndpi_flow = pointer(NDPIFlowStruct()) - memset(ndpi_flow, 0, sizeof(NDPIFlowStruct)) - if ip_src > ip_dst: - flow = WorkFlow(ip_src, ip_dst, src_port, dst_port, int(ip_packet.proto), 0, NDPIProtocol(), 0, 0, - pointer(NDPIIdStruct()), pointer(NDPIIdStruct()), ndpi_flow) - else: - flow = WorkFlow(ip_dst, ip_src, dst_port, src_port, int(ip_packet.proto), 0, NDPIProtocol(), 0, 0, - pointer(NDPIIdStruct()), pointer(NDPIIdStruct()), ndpi_flow) - flow_ref = pointer(flow) - res = ndpi.ndpi_tfind(flow_ref, flows_root_ref, cmp_func) - if res is None: - ndpi.ndpi_tsearch(flow_ref, pointer(flows_root), cmp_func) # add - lista.append(flow) - flow_count += 1 - return pointer(flow) - flow = cast(res, POINTER(POINTER(WorkFlow))).contents - return flow +ppacket = namedtuple('ParsedPacket', ['src_ip', + 'src_port', + 'dst_ip', + 'dst_port', + 'protocol', + 'ip_version', + 'ip_bytes']) -def packetcaptured(pkt): - global packet_number - global ndpi_info_mod - - flow = None - h = PcapPktHdr() - - # getting flow +def inet_to_str(inet): + """ get string representation of IP address """ try: - flow = get_flow(pkt) - except AttributeError: - pass # ignore packet - if flow is None: return - - # filling pcap_pkthdr - h.len = h.caplen = len(pkt) - h.ts.tv_sec = int(pkt["IP"].time/1000000) - h.ts.tv_usec = int(pkt["IP"].time) - - # real work - if int(pkt[1].frag) == 0: # not fragmented packet - flow.contents.packets += 1 - packet_number += 1 - # get ndpi_iphdr address - iphdr_addr = cast(c_char_p(pkt[1].build()), c_void_p) - ndpi_flow = flow.contents.flow - - if flow.contents.detection_completed is 0: - flow.contents.detected_protocol = ndpi.ndpi_detection_process_packet(ndpi_info_mod, - ndpi_flow, - cast(iphdr_addr, POINTER(c_uint8)), - int(pkt[1].len), - h.ts.tv_usec, - flow.contents.src_id, - flow.contents.dst_id) - - flow1 = flow.contents.detected_protocol - - valid = False - - if flow.contents.protocol == 6: valid = flow.contents.packets > max_num_tcp_dissected_pkts - elif flow.contents.protocol == 17: valid = flow.contents.packets > max_num_udp_dissected_pkts - - # should we continue anylizing packet or not? - if valid or flow1.app_protocol is not 0: - if valid or flow1.master_protocol is not 91: # or # 91 is NDPI_PROTOCOL_TLS - flow.contents.detection_completed = 1 # protocol found - if flow1.app_protocol is 0: - flow.contents.detected_protocol = ndpi.ndpi_detection_giveup(ndpi_info_mod, - ndpi_flow, - 1, - cast(addressof(c_uint8(0)), - POINTER(c_uint8))) - - -def result(): - global flows_root_ref - global ndpi_info_mod - print('\nnumber of analyzed packet: ' + str(packet_number)) - print('number of flows: ' + str(flow_count)) - - ndpi.ndpi_twalk(flows_root_ref.contents, guess_walker, None) + return socket.inet_ntop(socket.AF_INET, inet) + except ValueError: + return socket.inet_ntop(socket.AF_INET6, inet) - print('\nDetected protocols:') - for i in range(0, ndpi.ndpi_get_num_supported_protocols(ndpi_info_mod)): - if count_protocol[i] > 0: - print("{}: {} packets".format( - cast(ndpi.ndpi_get_proto_name(ndpi_info_mod, i), c_char_p).value.decode('utf-8'), - str(count_protocol[i]))) - -def free(ndpi_struct): - ndpi.ndpi_tdestroy(flows_root, free_walk) - ndpi.ndpi_exit_detection_module(ndpi_struct) - - -def initialize(ndpi_struct): - all = NDPIProtocolBitMask() - ndpi.ndpi_wrap_NDPI_BITMASK_SET_ALL(pointer(all)) - ndpi.ndpi_set_protocol_detection_bitmask2(ndpi_struct, pointer(all)) - - -print('Using nDPI ' + cast(ndpi.ndpi_revision(), c_char_p).value.decode("utf-8")) - -initialize(ndpi_info_mod) - -if len(sys.argv) != 2: - print("\nUsage: ndpi_example.py <device>") - sys.exit(0) - -if "." in sys.argv[1]: - print('Reading pcap from file ' + sys.argv[1] + '...') - scapy_cap = None +def parse_packet(pkt): + """ parse packet and extract 5 tuple and IP bytes """ try: - scapy_cap = rdpcap(sys.argv[1]) - except FileNotFoundError: - print("\nFile not found") - except Scapy_Exception: - print("\nBad pcap") + l2 = dpkt.ethernet.Ethernet(pkt) + if isinstance(l2.data, dpkt.ip.IP): + ip_version = 4 + elif isinstance(l2.data, dpkt.ip6.IP6): + ip_version = 6 + else: + return + except dpkt.dpkt.NeedData: + return + + l3 = l2.data + stop_decoding = False + while not stop_decoding: + if isinstance(l3.data, dpkt.tcp.TCP): + l4 = l3.data + proto = "TCP" + stop_decoding = True + elif isinstance(l3.data, dpkt.udp.UDP): + l4 = l3.data + proto = "UDP" + stop_decoding = True + elif isinstance(l3.data, dpkt.ip6.IP6): + l3 = l3.data + else: + return + + return ppacket(src_ip=inet_to_str(l3.src), src_port=l4.sport, + dst_ip=inet_to_str(l3.dst), dst_port=l4.dport, + protocol=proto, ip_version=ip_version, + ip_bytes=bytes(l3)) + + +def ppkt_to_flow_key(ppkt): + """ create a consistent direction agnostic flow keyfrom a parsed packet """ + if ppkt.src_ip < ppkt.dst_ip: + k = FLOW_KEY.format(ppkt.protocol, ppkt.src_ip, ppkt.src_port, ppkt.dst_ip, ppkt.dst_port) else: - for packet in scapy_cap: - packetcaptured(packet) -else: - print('Capturing live traffic from device ' + sys.argv[1] + '...') - try: - sniff(iface=sys.argv[1], prn=packetcaptured) - except KeyboardInterrupt: - print('\nInterrupted\n') - except PermissionError: - sys.exit('\nRoot privilege required for live capture on interface: {}\n'.format(sys.argv[1])) - - -result() -free(ndpi_info_mod) + if ppkt.src_ip == ppkt.dst_ip: + if ppkt.src_port <= ppkt.dst_port: + k = FLOW_KEY.format(ppkt.protocol, ppkt.src_ip, ppkt.src_port, ppkt.dst_ip, ppkt.dst_port) + else: + k = FLOW_KEY.format(ppkt.protocol, ppkt.dst_ip, ppkt.dst_port, ppkt.src_ip, ppkt.src_port) + else: + k = FLOW_KEY.format(ppkt.protocol, ppkt.dst_ip, ppkt.dst_port, ppkt.src_ip, ppkt.src_port) + return k + + +def parse_arguments(): + parser = argparse.ArgumentParser() + parser.add_argument("input", help="input pcap file path") + parser.add_argument('-u', '--include-unknowns', action='store_true') + return parser.parse_args() + + +if __name__ == "__main__": + nDPI = NDPI() # As simple as that. :) + flow_cache = {} # We store the flows in a dictionary. + flow_count = 0 # Flow counter + print("Using nDPI {}".format(nDPI.revision)) + args = parse_arguments() + + with open(args.input, 'rb') as pcap_file: + capture = dpkt.pcap.Reader(pcap_file) # We use dpkt pcap capture handler + for time, packet in capture: + time_ms = int(time * 1000) # Convert packet timestamp to milliseconds + ppkt = parse_packet(packet) + if ppkt is not None: # If we succeed to parse the packet + key = ppkt_to_flow_key(ppkt) + try: # Try a Flow update + flow = flow_cache[key] + flow.detected_protocol = nDPI.process_packet(flow.ndpi_flow, ppkt.ip_bytes, time_ms) + flow.pkts += 1 + flow.bytes += len(packet) + except KeyError: # New Flow + flow = Flow() + flow.index = flow_count + flow_count += 1 + flow.ndpi_flow = NDPIFlow() # We create an nDPIFlow object per Flow + flow.detected_protocol = nDPI.process_packet(flow.ndpi_flow, ppkt.ip_bytes, time_ms) + flow.pkts += 1 + flow.bytes += len(packet) + flow_cache[key] = flow + + print(" Detected flows:") + unknown_flows = [] + for key, flow in flow_cache.items(): # Iterate over all flows in flow cache + if flow.detected_protocol.app_protocol == PROTOCOL_UNKNWON: # Didn't succeed to identigy it using DPI + flow.detected_protocol = nDPI.giveup(flow.ndpi_flow) # We try to guess it (port matching, LRU, etc.) + FLOW_EXPORT = FLOW_STR.format(flow.index, + key, + nDPI.protocol_name(flow.detected_protocol), + nDPI.protocol_category_name(flow.detected_protocol), + flow.ndpi_flow.confidence.name, + flow.pkts, + flow.bytes) + if flow.detected_protocol.app_protocol != PROTOCOL_UNKNWON: + print(FLOW_EXPORT) # We start by printing detected flows + else: + # Format it for later + unknown_flows.append(FLOW_EXPORT) + if args.include_unknowns: + print(" Unknown flows:") + for unknown_flow in unknown_flows: # Dump unknown flows + print(unknown_flow) diff --git a/python/ndpi_typestruct.py b/python/ndpi_typestruct.py deleted file mode 100644 index 52238c6d9..000000000 --- a/python/ndpi_typestruct.py +++ /dev/null @@ -1,780 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- - -""" -file: ndpi_typestruct.py -This file is part of nDPI. - -Copyright (C) 2011-19 - ntop.org -Copyright (C) 2019 - Zied Aouini <aouinizied@gmail.com> (Incremental improvements) - -nDPI is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License -as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. - -nDPI is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty -of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. - -You should have received a copy of the GNU General Public License along with nDPI. -If not, see <http://www.gnu.org/licenses/>. -""" - -from ctypes import CDLL, Structure, c_uint16, c_int, c_ulong, c_uint32, CFUNCTYPE, c_void_p, POINTER, c_char_p, c_uint8 -from ctypes import c_char, c_uint, c_int16, c_longlong, c_size_t, Union, c_ubyte, c_uint64, c_int32, c_ushort, cast -from os.path import abspath, dirname -ndpi = CDLL(dirname(abspath(__file__)) + '/ndpi_wrap.so') - -# ----------------------------------------------- Structures ----------------------------------------------------------- - - -class NDPIDetectionModuleStruct(Structure): - pass - - -class NDPIFlowStruct(Structure): - pass - - -class NDPIProtocol(Structure): - _fields_ = [ - ("master_protocol", c_uint16), - ("app_protocol", c_uint16), - ("category", c_int) - ] - - -class TimeVal(Structure): - _fields_ = [("tv_sec", c_ulong), ("tv_usec", c_ulong)] - - -class PcapPktHdr(Structure): - _fields_ = [("ts", TimeVal), ("caplen", c_uint32), ("len", c_uint32)] - - -class NDPIMask(Structure): - _fields_ = [("fds_bits", c_uint32)] - - -class NDPIProtocolBitMask(Structure): - _fields_ = [("fds_bits", NDPIMask * ndpi.ndpi_wrap_ndpi_num_fds_bits())] - - -class NDPISubprotocolConfStruct(Structure): - _fields_ = [("func", CFUNCTYPE(c_void_p, POINTER(NDPIDetectionModuleStruct), c_char_p, c_char_p, c_int))] - - -class NDPIAutoma(Structure): - _fields_ = [ - ("ac_automa", c_void_p), - ("ac_automa_finalized", c_uint8) - ] - - -class NDPINode(Structure): - pass - - -NDPINode._fields_ = [ - ('key', POINTER(c_char)), - ('left', POINTER(NDPINode)), - ('right', POINTER(NDPINode)), -] - - -class NDPICallFunctionStruct(Structure): - _fields_ = [ - ("detection_bitmask", NDPIProtocolBitMask), - ("excluded_protocol_bitmask", NDPIProtocolBitMask), - ("ndpi_selection_bitmask", c_uint32), - ("func", CFUNCTYPE(None, POINTER(NDPIDetectionModuleStruct), POINTER(NDPIFlowStruct))), - ("detection_feature", c_uint8) - ] - - -class NDPIProtoDefaultsT(Structure): - _fields_ = [ - ("protoName", c_char_p), - ("protoCategory", c_uint), - ("subprotocols", c_uint16_p), - ("subprotocol_count", c_uint32), - ("protoId", c_uint16), - ("protoIdx", c_uint16), - ("protoBreed", c_uint), - ("func", CFUNCTYPE(None, POINTER(NDPIDetectionModuleStruct), POINTER(NDPIFlowStruct))), - ] - - -class NDPIDefaultsPortsTreeNodeT(Structure): - _fields_ = [ - ("proto", NDPIProtoDefaultsT), - ("customUserProto", c_uint8), - ("default_port", c_int16) - ] - - -class SpinlockT(Structure): - _fields_ = [("val", c_int)] - - -class AtomicT(Structure): - _fields_ = [("counter", c_int)] - - -class TimeT(Structure): - _fields_ = [("counter", c_longlong)] - - -class HashIp4pNode(Structure): - pass - - -HashIp4pNode._fields_ = [ - ("next", POINTER(HashIp4pNode)), - ("prev", POINTER(HashIp4pNode)), - ("lchg", TimeT), - ("port", c_uint16), - ("count", c_uint16, 12), - ("flag", c_uint16, 4), - ("ip", c_uint32) -] - - -class HashIp4p(Structure): - _fields_ = [ - ("top", POINTER(HashIp4pNode)), - ("lock",SpinlockT), - ("len", c_size_t) - ] - - -class HashIp4pTable(Structure): - _fields_ = [ - ("size", c_size_t), - ("ipv6", c_int), - ("lock", SpinlockT), - ("count", AtomicT), - ("tbl", HashIp4p) - ] - - -class BtAnnounce(Structure): - _fields_ = [ - ("hash", c_uint32 * 5), - ("ip", c_uint32 * 4), - ("time", c_uint32), - ("port", c_uint16), - ("name_len", c_uint8), - ("name", c_uint8 * 149) - ] - - -class NDPILruCacheEntry(Structure): - _fields_ = [ - ("key", c_uint32), - ("is_full", c_uint32, 1), - ("value", c_uint32, 16), - ("pad", c_uint32, 15) - ] - - -class NDPILruCache(Structure): - _fields_ = [ - ("num_entries", c_uint32), - ("entries", POINTER(NDPILruCacheEntry)), - ] - - -class CacheEntry(Structure): - pass - - -CacheEntry._fields_ = [ - ("item", c_void_p), - ("item_size", c_uint32), - ("prev", POINTER(CacheEntry)), - ("next", POINTER(CacheEntry)) -] - - -class CacheEntryMap(Structure): - pass - - -CacheEntryMap._fields_ = [ - ("entry", POINTER(CacheEntry)), - ("next", POINTER(CacheEntryMap)), -] - - -class Cache(Structure): - _fields_ = [ - ("size", c_uint32), - ("max_size", c_uint32), - ("head", POINTER(CacheEntry)), - ("tail", POINTER(CacheEntry)), - ("map", POINTER(POINTER(CacheEntryMap))) - ] - - -class CustomCategories(Structure): - _fields_ = [ - ("hostnames", NDPIAutoma), - ("hostnames_shadow", NDPIAutoma), - ("ipAddresses", c_void_p), - ("ipAddresses_shadow", c_void_p), - ("categories_loaded", c_uint8), - ] - - -NDPIDetectionModuleStruct._fields_ = [ - ("detection_bitmask", NDPIProtocolBitMask), - ("current_ts", c_uint32), - ("custom_category_labels", - (c_char * ndpi.ndpi_wrap_num_custom_categories()) * ndpi.ndpi_wrap_custom_category_label_len()), - ("callback_buffer", NDPICallFunctionStruct * (ndpi.ndpi_wrap_ndpi_max_supported_protocols() + 1)), - ("callback_buffer_size", c_uint32), - ("callback_buffer_tcp_no_payload", NDPICallFunctionStruct * (ndpi.ndpi_wrap_ndpi_max_supported_protocols() + 1)), - ("callback_buffer_size_tcp_no_payload", c_uint32), - ("callback_buffer_tcp_payload", NDPICallFunctionStruct * (ndpi.ndpi_wrap_ndpi_max_supported_protocols() + 1)), - ("callback_buffer_size_tcp_payload", c_uint32), - ("callback_buffer_udp", NDPICallFunctionStruct * (ndpi.ndpi_wrap_ndpi_max_supported_protocols() + 1)), - ("callback_buffer_size_udp", c_uint32), - ("callback_buffer_non_tcp_udp", NDPICallFunctionStruct * (ndpi.ndpi_wrap_ndpi_max_supported_protocols() + 1)), - ("callback_buffer_size_non_tcp_udp", c_uint32), - ("tcpRoot", POINTER(NDPIDefaultsPortsTreeNodeT)), - ("udpRoot", POINTER(NDPIDefaultsPortsTreeNodeT)), - ("ndpi_log_level", c_uint), - ("tcp_max_retransmission_window_size", c_uint32), - ("subprotocol_conf", NDPISubprotocolConfStruct * (ndpi.ndpi_wrap_ndpi_max_supported_protocols() + 1)), - ("ndpi_num_supported_protocols", c_uint), - ("ndpi_num_custom_protocols", c_uint), - ("host_automa", NDPIAutoma), - ("bigrams_automa", NDPIAutoma), - ("impossible_bigrams_automa", NDPIAutoma), - ("custom_categories", CustomCategories), - ("protocols_ptree", c_void_p), - ("ip_version_limit", c_uint8), - ("ookla_cache", POINTER(NDPILruCache)), - ("tinc_cache", POINTER(Cache)), - ("proto_defaults", NDPIProtoDefaultsT * (ndpi.ndpi_wrap_ndpi_max_supported_protocols() + - ndpi.ndpi_wrap_ndpi_max_num_custom_protocols())), - ("direction_detect_disable", c_uint8, 1), - ('_pad', c_uint8, 7), - ('packet', NDPIPacketStruct), -] - - -class U6Addr(Union): - _fields_ = [ - ("u6_addr8", c_uint8 * 16), - ("u6_addr16", c_uint16 * 8), - ("u6_addr32", c_uint32 * 4), - ("u6_addr64", c_uint64 * 2) - ] - - -class NDPIIn6Addr(Structure): - _pack_ = 1 - _fields_ = [("u6_addr", U6Addr)] - - -class NDPIIpAddrT(Union): - _fields_ = [ - ('ipv4', c_uint32), - ('ipv4_u_int8_t', c_uint8 * 4), - ('ipv6', NDPIIn6Addr), - ] - - -class NDPIIdStruct(Structure): - _fields_ = [ - ('detected_protocol_bitmask', NDPIProtocolBitMask), - ('irc_port', c_uint16 * 8), - ('last_time_port_used', c_uint32 * 8), - ('irc_ts', c_uint32), - ('gnutella_ts', c_uint32), - ('battlefield_ts', c_uint32), - ('rtsp_timer', c_uint32), - ('jabber_stun_or_ft_ts', c_uint32), - ('directconnect_last_safe_access_time', c_uint32), - ('detected_directconnect_port', c_uint16), - ('detected_directconnect_udp_port', c_uint16), - ('detected_directconnect_ssl_port', c_uint16), - ('jabber_voice_stun_port', c_uint16 * 6), - ('jabber_file_transfer_port', c_uint16 * 2), - ('detected_gnutella_udp_port1', c_uint16), - ('detected_gnutella_udp_port2', c_uint16), - ('irc_number_of_port', c_uint8), - ('oscar_ssl_session_id', c_uint8 * 33), - ('jabber_voice_stun_used_ports', c_uint8), - ] - - -class NDPIFlowTcpStruct(Structure): - _pack_ = 1 - _fields_ = [ - ('smtp_command_bitmask', c_uint16), - ('pop_command_bitmask', c_uint16), - ('wa_matched_so_far', c_uint8), - ('irc_stage', c_uint8), - ('h323_valid_packets', c_uint8), - ('gnutella_msg_id', c_uint8 * 3), - ('irc_3a_counter', c_uint32, 3), - ('irc_stage2', c_uint32, 5), - ('irc_direction', c_uint32, 2), - ('irc_0x1000_full', c_uint32, 1), - ('usenet_stage', c_uint32, 2), - ('http_stage', c_uint32, 2), - ('http_empty_line_seen', c_uint32, 1), - ('gnutella_stage', c_uint32, 2), - ('ssh_stage', c_uint32, 3), - ('vnc_stage', c_uint32, 2), - ('telnet_stage', c_uint32, 2), - ('tls_seen_client_cert', c_uint8, 1), - ('tls_seen_server_cert', c_uint8, 1), - ('tls_seen_certificate', c_uint8, 1), - ('tls_srv_cert_fingerprint_found', c_uint8, 1), - ('tls_srv_cert_fingerprint_processed', c_uint8, 1), - ('tls_stage', c_uint8, 2), - ('tls_record_offset', c_int16), - ('tls_fingerprint_len', c_int16), - ('tls_sha1_certificate_fingerprint', c_uint8 * 20), - ('postgres_stage', c_uint32, 3), - ('seen_syn', c_uint32, 1), - ('seen_syn_ack', c_uint32, 1), - ('seen_ack', c_uint32, 1), - ('icecast_stage', c_uint32, 1), - ('dofus_stage', c_uint32, 1), - ('fiesta_stage', c_uint32, 2), - ('wow_stage', c_uint32, 2), - ('shoutcast_stage', c_uint32, 2), - ('rtp_special_packets_seen', c_uint32, 1), - ('mail_pop_stage', c_uint32, 2), - ('mail_imap_stage', c_uint32, 3), - ('mail_imap_starttls', c_uint32, 2), - ('skype_packet_id', c_uint8), - ('lotus_notes_packet_id', c_uint8), - ('teamviewer_stage', c_uint8), - ('prev_zmq_pkt_len', c_uint8), - ('prev_zmq_pkt', c_char * 10), - ('ppstream_stage', c_uint32, 3), - ('memcached_matches', c_uint8), - ('nest_log_sink_matches', c_uint8), - ] - - -class NDPIFlowUdpStruct(Structure): - _pack_ = 1 - _fields_ = [ - ('ppstream_stage', c_uint32, 3), - ('halflife2_stage', c_uint32, 2), - ('tftp_stage', c_uint32, 2), - ('aimini_stage', c_uint32, 5), - ('xbox_stage', c_uint32, 1), - ('skype_packet_id', c_uint8), - ('teamviewer_stage', c_uint8), - ('eaq_pkt_id', c_uint8), - ('eaq_sequence', c_uint32), - ('rx_conn_epoch', c_uint32), - ('rx_conn_id', c_uint32), - ('memcached_matches', c_uint8), - ('wireguard_stage', c_uint8), - ('wireguard_peer_index', c_uint32 * 2), - ('quic_reasm_buf', POINTER(c_uint8)), - ('quic_reasm_buf_len', c_uint32), - ('csgo_strid', c_uint8 * 18), - ('csgo_state', c_uint8), - ('csgo_s2', c_uint8), - ('csgo_id2', c_uint32), - ('rdp_to_srv', c_uint8 * 3), - ('rdp_from_srv', c_uint8 * 3), - ('rdp_to_srv_pkts,', c_uint8), - ('rdp_from_srv_pkts', c_uint8), - ('imo_last_one_byte_pkt,', c_uint8), - ('imo_last_byte', c_uint8), - ] - - -class L4(Union): - _fields_ = [("tcp", NDPIFlowTcpStruct), ("udp", NDPIFlowUdpStruct)] - - -class Http(Structure): - _fields_ = [ - ("method", c_int), - ("request_version", c_uint8), - ("response_status_code", c_uint16), - ("url", c_char_p), - ("content_type", c_char_p), - ("user_agent", c_char_p), - ("detected_os", c_char_p), - ("nat_ip", c_char_p), - ] - - -class Dns(Structure): - _fields_ = [ - ("num_queries", c_uint8), - ("num_answers", c_uint8), - ("reply_code", c_uint8), - ("is_query", c_uint8), - ("query_type", c_uint16), - ("query_class", c_uint16), - ("rsp_type", c_uint16), - ("rsp_addr", NDPIIpAddrT) - ] - - -class Ntp(Structure): - _fields_ = [("request_code", c_uint8), - ("version", c_uint8)] - - -class Kerberos(Structure): - _fields_ = [("cname", c_char * 24), - ("realm", c_char * 24)] - - -class QuicSsl(Structure): - _fields_ = [ - ("ssl_version", c_uint16), - ("client_certificate", c_char * 64), - ("server_certificate", c_char * 64), - ("server_organization", c_char * 64), - ('notBefore', c_uint32), - ('notAfter', c_uint32), - ("ja3_client", c_char * 33), - ("ja3_server", c_char * 33), - ("server_cipher", c_uint16), - ("server_unsafe_cipher", c_int) - ] - - -class Stun(Structure): - _fields_ = [ - ("num_udp_pkts", c_uint8), - ("num_processed_pkts", c_uint8), - ("num_binding_requests", c_uint8), - ] - - -class Ssh(Structure): - _fields_ = [ - ("client_signature", c_char * 48), - ("server_signature", c_char * 48), - ("hassh_client", c_char * 33), - ("hassh_server", c_char * 33) - ] - - -class Mdns(Structure): - _fields_ = [("answer", c_char * 96)] - - -class Ubntac2(Structure): - _fields_ = [("version", c_char * 32)] - - -class FtpImapPopSmtp(Structure): - _fields_ = [ - ("auth_found", c_uint8, 1), - ("auth_failed", c_uint8, 1), - ("auth_tls", c_uint8, 1), - ("auth_done", c_uint8, 1), - ("_pad", c_uint8, 4), - ("username", c_char * 32), - ("password", c_char * 16) - ] - -class Bittorrent(Structure): - _fields_ = [("hash", c_char * 20)] - - -class Dhcp(Structure): - _fields_ = [ - ("fingerprint", c_char * 48), - ("class_ident", c_char * 48) - ] - - -class Protos(Union): - _fields_ = [ - ("dns", Dns), - ("kerberos", Kerberos), - ("quic_ssl", QuicSsl), - ("ssh", Ssh), - ("mdns", Mdns), - ("ubntac2", Ubntac2), - ("bittorrent", Bittorrent), - ("dhcp", Dhcp) - ] - - -class TincCacheEntry(Structure): - _pack_ = 1 - _fields_ = [ - ('src_address', c_uint32), - ('dst_address', c_uint32), - ('dst_port', c_uint16), - ] - - -class NDPIIntOneLineStruct(Structure): - _fields_ = [ - ('ptr', POINTER(c_uint8)), - ('len', c_uint16), - ] - - -class NDPIIphdr(Structure): - _pack_ = 1 - _fields_ = [ - ('ihl', c_uint8, 4), - ('version', c_uint8, 4), - ('tos', c_uint8), - ('tot_len', c_uint16), - ('id', c_uint16), - ('frag_off', c_uint16), - ('ttl', c_uint8), - ('protocol', c_uint8), - ('check', c_uint16), - ('saddr', c_uint32), - ('daddr', c_uint32)] - - -class NDPIIp6Hdrctl(Structure): - _pack_ = 1 - _fields_ = [ - ('ip6_un1_flow', c_uint32), - ('ip6_un1_plen', c_uint16), - ('ip6_un1_nxt', c_uint8), - ('ip6_un1_hlim', c_uint8), - ] - - -class NDPIIpv6hdr(Structure): - _pack_ = 1 - _fields_ = [ - ('ip6_hdr', NDPIIp6Hdrctl), - ('ip6_src', NDPIIn6Addr), - ('ip6_dst', NDPIIn6Addr), - ] - - -class NDPITcpHdr(Structure): - _pack_ = 1 - _fields_ = [ - ('source', c_uint16), - ('dest', c_uint16), - ('seq', c_uint32), - ('ack_seq', c_uint32), - ('res1', c_uint16, 4), - ('doff', c_uint16, 4), - ('fin', c_uint16, 1), - ('syn', c_uint16, 1), - ('rst', c_uint16, 1), - ('psh', c_uint16, 1), - ('ack', c_uint16, 1), - ('urg', c_uint16, 1), - ('ece', c_uint16, 1), - ('cwr', c_uint16, 1), - ('window', c_uint16), - ('check', c_uint16), - ('urg_ptr', c_uint16), - ] - - -class NDPIUdpHdr(Structure): - _pack_ = 1 - _fields_ = [ - ('source', c_uint16), - ('dest', c_uint16), - ('len', c_uint16), - ('check', c_uint16), - ] - - -class NDPIPacketStruct(Structure): - _fields_ = [ - ('iph', POINTER(NDPIIphdr)), - ('iphv6', POINTER(NDPIIpv6hdr)), - ('tcp', POINTER(NDPITcpHdr)), - ('udp', POINTER(NDPIUdpHdr)), - ('generic_l4_ptr', POINTER(c_uint8)), - ('payload', POINTER(c_uint8)), - ('current_timestamp_ms', c_uint64), - ('detected_protocol_stack', c_uint16 * ndpi.ndpi_wrap_ndpi_procol_size()), - ('line', NDPIIntOneLineStruct * 64), - ('host_line', NDPIIntOneLineStruct), - ('forwarded_line', NDPIIntOneLineStruct), - ('referer_line', NDPIIntOneLineStruct), - ('content_line', NDPIIntOneLineStruct), - ('accept_line', NDPIIntOneLineStruct), - ('user_agent_line', NDPIIntOneLineStruct), - ('http_url_name', NDPIIntOneLineStruct), - ('http_encoding', NDPIIntOneLineStruct), - ('http_transfer_encoding', NDPIIntOneLineStruct), - ('http_contentlen', NDPIIntOneLineStruct), - ('http_cookie', NDPIIntOneLineStruct), - ('http_origin', NDPIIntOneLineStruct), - ('http_x_session_type', NDPIIntOneLineStruct), - ('server_line', NDPIIntOneLineStruct), - ('http_method', NDPIIntOneLineStruct), - ('http_response', NDPIIntOneLineStruct), - ('http_num_headers', c_uint8), - ('l3_packet_len', c_uint16), - ('payload_packet_len', c_uint16), - ('parsed_lines', c_uint16), - ('empty_line_position', c_uint16), - ('tcp_retransmission', c_uint8), - ('packet_lines_parsed_complete', c_uint8, 1), - ('packet_direction', c_uint8, 1), - ('empty_line_position_set', c_uint8, 1), - ('http_check_content', c_uint8, 1), - ('pad', c_uint8, 4), - ] - - -class NDPIFlowStructStack(Structure): - _pack_ = 1 - _fields_ = [ - ("detected_protocol_stack", c_uint16 * ndpi.ndpi_wrap_ndpi_procol_size()), - ] - - -NDPIFlowStruct._fields_ = [ - ("ndpi_flow_stack", NDPIFlowStructStack), - ("guessed_protocol_id", c_uint16), - ("guessed_host_protocol_id", c_uint16), - ("guessed_category", c_uint16), - ("guessed_header_category", c_uint16), - ("l4_proto", c_uint8), - ("protocol_id_already_guessed", c_uint8, 1), - ("host_already_guessed", c_uint8, 1), - ("init_finished", c_uint8, 1), - ("setup_packet_direction", c_uint8, 1), - ("packet_direction", c_uint8, 1), - ("check_extra_packets", c_uint8, 1), - ("next_tcp_seq_nr", c_uint32 * 2), - ("max_extra_packets_to_check", c_uint8), - ("num_extra_packets_checked", c_uint8), - ("num_processed_pkts", c_uint8), - ("extra_packets_func", CFUNCTYPE(c_int, POINTER(NDPIDetectionModuleStruct), POINTER(NDPIFlowStruct))), - ("l4", L4), - ("host_server_name", c_char * 80), - ("http", Http), - ("stun", Stun), - ("ftp_imap_pop_smtp", FtpImapPopSmtp), - ("protos", Protos), - ("excluded_protocol_bitmask", NDPIProtocolBitMask), - ("category", c_int), - ('redis_s2d_first_char', c_uint8), - ('redis_d2s_first_char', c_uint8), - ('packet_counter', c_uint16), - ('packet_direction_counter', c_uint16 * 2), - ('byte_counter', c_uint16 * 2), - ('bittorrent_stage', c_uint8), - ('directconnect_stage', c_uint8, 2), - ('http_detected', c_uint8, 1), - ('http_upper_protocol', c_uint16), - ('http_lower_protocol', c_uint16), - ('rtsprdt_stage', c_uint8, 2), - ('zattoo_stage', c_uint8, 3), - ('thunder_stage', c_uint8, 2), - ('oscar_ssl_voice_stage', c_uint8, 3), - ('oscar_video_voice', c_uint8, 1), - ('florensia_stage', c_uint8, 1), - ('socks5_stage', c_uint8, 2), - ('socks4_stage', c_uint8, 2), - ('edonkey_stage', c_uint8, 2), - ('ftp_control_stage', c_uint8, 2), - ('rtmp_stage', c_uint8, 2), - ('pando_stage', c_uint8, 3), - ('steam_stage1', c_uint16, 3), - ('steam_stage2', c_uint16, 2), - ('steam_stage3', c_uint16, 2), - ('pplive_stage1', c_uint8, 3), - ('pplive_stage2', c_uint8, 2), - ('pplive_stage3', c_uint8, 2), - ('starcraft_udp_stage', c_uint8, 3), - ('ovpn_session_id', c_uint8 * 8), - ('ovpn_counter', c_uint8), - ('key_mining_cache', c_uint32), - ('tinc_state', c_uint8), - ('TincCacheEntry', TincCacheEntry), - ('src', POINTER(NDPIIdStruct)), - ('dst', POINTER(NDPIIdStruct)) -] - -# ----------------------------------------------- nDPI APIs ------------------------------------------------------------ - -""" ndpi_detection_giveup: Function to be called before we give up with detection for a given flow. - This function reduces the NDPI_UNKNOWN_PROTOCOL detection. """ -ndpi.ndpi_detection_giveup.restype = NDPIProtocol -ndpi.ndpi_detection_giveup.argtypes = [POINTER(NDPIDetectionModuleStruct), - POINTER(NDPIFlowStruct), c_uint8, - POINTER(c_uint8)] - -""" ndpi_detection_process_packet: Processes one packet and returns the ID of the detected protocol. - This is the MAIN PACKET PROCESSING FUNCTION. """ -ndpi.ndpi_detection_process_packet.restype = NDPIProtocol -ndpi.ndpi_detection_process_packet.argtypes = [POINTER(NDPIDetectionModuleStruct), - POINTER(NDPIFlowStruct), - POINTER(c_ubyte), - c_ushort, - c_uint64, - POINTER(NDPIIdStruct), - POINTER(NDPIIdStruct)] - -""" ndpi_ssl_version2str : Converts ssl version to readable string """ -ndpi.ndpi_ssl_version2str.restype = c_char_p -ndpi.ndpi_ssl_version2str.argtypes = [c_int16, POINTER(c_uint8)] - -""" ndpi_init_detection_module: Returns a new initialized detection module. - Note that before you can use it you can still load hosts and do other things. As soon as you are ready to use - it do not forget to call first ndpi_finalize_initalization() """ -ndpi.ndpi_init_detection_module.restype = POINTER(NDPIDetectionModuleStruct) - - -def ndpi_ndpi_finalize_initalization(detection_module): - """ ndpi_finalize_initalization: Completes the initialization (ndpi_revision >= 3.1)""" - if cast(ndpi.ndpi_revision(), c_char_p).value.decode("utf-8")[:3] >= '3.1': - ndpi.ndpi_finalize_initalization.restype = c_void_p - ndpi.ndpi_finalize_initalization.argtypes = [POINTER(NDPIDetectionModuleStruct)] - return ndpi.ndpi_finalize_initalization(detection_module) - else: - # ignore it - return None - - -""" ndpi_tfind: find a node, or return 0. """ -ndpi.ndpi_tfind.restype = c_void_p - -""" ndpi_tsearch: ftp://ftp.cc.uoc.gr/mirrors/OpenBSD/src/lib/libc/stdlib/tsearch.c - find or insert datum into search tree. """ -ndpi.ndpi_tsearch.restype = c_void_p -ndpi.ndpi_tsearch.argtypes = [c_void_p, POINTER(c_void_p), CFUNCTYPE(c_int, c_void_p, c_void_p)] - -""" ndpi_revision: Get the nDPI version release. """ -ndpi.ndpi_revision.restype = c_void_p - -""" ndpi_get_proto_name: Get the protocol name associated to the ID.""" -ndpi.ndpi_get_proto_name.restype = c_void_p - -""" ndpi_category_get_name: Get protocol category as string.""" -ndpi.ndpi_category_get_name.restype = c_void_p - -""" ndpi_get_num_supported_protocols: Get the total number of the supported protocols.""" -ndpi.ndpi_get_num_supported_protocols.restype = c_uint - -""" ndpi_wrap_NDPI_BITMASK_SET_ALL: memset((char *)(p), 0xFF, sizeof(*(p)))""" -ndpi.ndpi_wrap_NDPI_BITMASK_SET_ALL.argtypes = [POINTER(NDPIProtocolBitMask)] - -""" ndpi_set_protocol_detection_bitmask2: Sets the protocol bitmask2.""" -ndpi.ndpi_set_protocol_detection_bitmask2.argtypes = [POINTER(NDPIDetectionModuleStruct), - POINTER(NDPIProtocolBitMask)] - -""" ndpi_twalk: Walk the nodes of a tree. """ -ndpi.ndpi_twalk.argtypes = [c_void_p, CFUNCTYPE(None, c_void_p, c_int32, c_int, c_void_p), c_void_p] - -""" ndpi_tdestroy: node destroy. """ -ndpi.ndpi_tdestroy.argtypes = [c_void_p, CFUNCTYPE(None, c_void_p)] diff --git a/python/ndpi_wrap.c b/python/ndpi_wrap.c deleted file mode 100644 index 7a052ddff..000000000 --- a/python/ndpi_wrap.c +++ /dev/null @@ -1,60 +0,0 @@ -/* - * ndpi_wrap.c - * - * Copyright (C) 2011-19 - ntop.org - * - * nDPI is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * nDPI is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with nDPI. If not, see <http://www.gnu.org/licenses/>. - * - */ - -#include "sys/types.h" -#include "ndpi_config.h" -#include "ndpi_main.h" - -int ndpi_wrap_get_api_version(){ - return NDPI_API_VERSION; -} - -int ndpi_wrap_ndpi_num_fds_bits(){ - return NDPI_NUM_FDS_BITS; -} - -int ndpi_wrap_num_custom_categories(){ - return NUM_CUSTOM_CATEGORIES; -} - -int ndpi_wrap_custom_category_label_len(){ - return CUSTOM_CATEGORY_LABEL_LEN; -} - -int ndpi_wrap_ndpi_max_supported_protocols(){ - return NDPI_MAX_SUPPORTED_PROTOCOLS; -} - -int ndpi_wrap_ndpi_max_num_custom_protocols(){ - return NDPI_MAX_NUM_CUSTOM_PROTOCOLS; -} - -int ndpi_wrap_ndpi_procol_size(){ - return NDPI_PROTOCOL_SIZE; -} - -void ndpi_wrap_NDPI_BITMASK_SET_ALL(NDPI_PROTOCOL_BITMASK* bitmask){ - NDPI_ONE(bitmask); -} - -void dummy() { - /* Dummy call just to cause linker to include the ndpi library */ - ndpi_tfind(NULL, NULL, NULL); -} diff --git a/python/python_extensions_guide.pdf b/python/python_extensions_guide.pdf Binary files differdeleted file mode 100644 index 10c4f08b6..000000000 --- a/python/python_extensions_guide.pdf +++ /dev/null diff --git a/python/requirements.txt b/python/requirements.txt new file mode 100644 index 000000000..5ccde2971 --- /dev/null +++ b/python/requirements.txt @@ -0,0 +1 @@ +cffi>=1.15.0
\ No newline at end of file diff --git a/python/setup.py b/python/setup.py new file mode 100644 index 000000000..5410d4529 --- /dev/null +++ b/python/setup.py @@ -0,0 +1,57 @@ +""" +------------------------------------------------------------------------------------------------------------------------ +setup.py +Copyright (C) 2011-22 - ntop.org +This file is part of nDPI, an open source deep packet inspection library. +nDPI is free software: you can redistribute it and/or modify it under the terms of the GNU Lesser General Public +License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later +version. +nDPI is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details. +You should have received a copy of the GNU Lesser General Public License along with NFStream. +If not, see <http://www.gnu.org/licenses/>. +------------------------------------------------------------------------------------------------------------------------ +""" + +from setuptools import setup +import os + + +this_directory = os.path.abspath(os.path.dirname(__file__)) +with open(os.path.join(this_directory, 'README.md'), encoding='utf-8') as f: + long_description = f.read() + + +setup( + name="ndpi", + version='4.3.0', + url='https://www.ntop.org/products/deep-packet-inspection/ndpi/', + license='LGPLv3', + description="Open and Extensible LGPLv3 Deep Packet Inspection Library", + long_description=long_description, + long_description_content_type='text/markdown', + author='Zied Aouini', + author_email='aouinizied@gmail.com', + packages=['ndpi'], + setup_requires=["cffi>=1.15.0"], + cffi_modules=["ndpi/ndpi_build.py:ffi_builder"], + install_requires=["cffi>=1.15.0"], + platforms=["Linux", + "Mac OS-X", + "Windows", + "Unix"], + classifiers=[ + 'Development Status :: 5 - Production/Stable', + 'License :: OSI Approved :: GNU Lesser General Public License v3 or later (LGPLv3+)', + 'Programming Language :: Python :: 3 :: Only', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: 3.10', + 'Topic :: System :: Networking :: Monitoring', + ], + project_urls={ + 'GitHub': 'https://github.com/ntop/nDPI', + } +) diff --git a/python/tests.py b/python/tests.py new file mode 100644 index 000000000..ebb4abbb9 --- /dev/null +++ b/python/tests.py @@ -0,0 +1,28 @@ +""" +------------------------------------------------------------------------------------------------------------------------ +tests.py +Copyright (C) 2011-22 - ntop.org +This file is part of nDPI, an open source deep packet inspection library. +nDPI is free software: you can redistribute it and/or modify it under the terms of the GNU Lesser General Public +License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later +version. +nDPI is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details. +You should have received a copy of the GNU Lesser General Public License along with NFStream. +If not, see <http://www.gnu.org/licenses/>. +------------------------------------------------------------------------------------------------------------------------ +""" + +from ndpi import NDPI, NDPIFlow +import time + + +if __name__ == '__main__': + try: + nDPI = NDPI() + ndpi_flow = NDPIFlow() + nDPI.process_packet(ndpi_flow, b'', time.time()) + nDPI.giveup(ndpi_flow) + print("nDPI Python bindings: OK") + except Exception: + raise AssertionError("nDPI Python bindings: KO") |