From 9db048c9d93a00adf4b258d2341b24229d2a45a1 Mon Sep 17 00:00:00 2001 From: Toni Uhlig Date: Sun, 27 Feb 2022 02:53:39 +0100 Subject: Serialize flow risk score / confidence. * bump libnDPI to 8b062295cc76a60e3905c054ce37bd17669464d1 * removed ndpi_id_struct's Signed-off-by: Toni Uhlig --- CMakeLists.txt | 24 +- dependencies/nDPIsrvd.h | 35 +- dependencies/nDPIsrvd.py | 17 +- examples/py-flow-dashboard/flow-dash.py | 16 +- examples/py-flow-info/flow-info.py | 12 +- .../py-flow-muliprocess/py-flow-multiprocess.py | 11 +- examples/py-ja3-checker/py-ja3-checker.py | 10 +- examples/py-json-stdout/json-stdout.py | 10 +- .../py-schema-validation/py-schema-validation.py | 12 +- .../py-semantic-validation.py | 11 +- libnDPI | 2 +- nDPId-test.c | 6 + nDPId.c | 107 +- scripts/get-and-build-libndpi.sh | 11 + test/results/1kxun.pcap.out | 462 +++--- test/results/443-chrome.pcap.out | 8 +- test/results/443-curl.pcap.out | 18 +- test/results/443-firefox.pcap.out | 18 +- test/results/443-git.pcap.out | 18 +- test/results/443-opvn.pcap.out | 14 +- test/results/443-safari.pcap.out | 18 +- test/results/4in4tunnel.pcap.out | 6 +- test/results/4in6tunnel.pcap.out | 12 +- test/results/6in4tunnel.pcap.out | 6 +- test/results/6in6tunnel.pcap.out | 6 +- test/results/BGP_Cisco_hdlc_slarp.pcap.out | 14 +- test/results/BGP_redist.pcap.out | 14 +- test/results/EAQ.pcap.out | 148 +- test/results/IEC104.pcap.out | 18 +- test/results/KakaoTalk_chat.pcap.out | 206 +-- test/results/KakaoTalk_talk.pcap.out | 76 +- test/results/NTPv2.pcap.out | 10 +- test/results/NTPv3.pcap.out | 14 +- test/results/NTPv4.pcap.out | 14 +- test/results/Oscar.pcap.out | 16 +- test/results/WebattackRCE.pcap.out | 1604 ++++++++++---------- test/results/WebattackSQLinj.pcap.out | 46 +- test/results/WebattackXSS.pcap.out | 1376 ++++++++--------- test/results/aimini-http.pcap.out | 26 +- test/results/ajp.pcap.out | 14 +- test/results/alexa-app.pcapng.out | 848 +++++------ test/results/among_us.pcap.out | 14 +- test/results/amqp.pcap.out | 18 +- test/results/android.pcap.out | 324 ++-- test/results/anyconnect-vpn.pcap.out | 314 ++-- test/results/anydesk-2.pcap.out | 30 +- test/results/anydesk.pcap.out | 20 +- test/results/avast_securedns.pcapng.out | 170 +-- test/results/bad-dns-traffic.pcap.out | 42 +- test/results/badpackets.pcap.out | 6 +- test/results/bitcoin.pcap.out | 34 +- test/results/bittorrent.pcap.out | 102 +- test/results/bittorrent_ip.pcap.out | 31 - test/results/bittorrent_utp.pcap.out | 16 +- test/results/bot.pcap.out | 23 + test/results/bt_search.pcap.out | 14 +- test/results/capwap.pcap.out | 32 +- test/results/cassandra.pcap.out | 18 +- test/results/check_mk_new.pcap.out | 14 +- test/results/chrome.pcap.out | 46 +- test/results/coap_mqtt.pcap.out | 76 +- test/results/cpha.pcap.out | 14 +- test/results/dcerpc.pcap.out | 22 +- test/results/dhcp-fuzz.pcapng.out | 8 +- test/results/diameter.pcap.out | 10 +- test/results/dlt_ppp.pcap.out | 6 +- test/results/dnp3.pcap.out | 50 +- test/results/dns-invalid-chars.pcap.out | 16 +- test/results/dns-tunnel-iodine.pcap.out | 16 +- test/results/dns_ambiguous_names.pcap.out | 66 +- test/results/dns_doh.pcap.out | 16 +- test/results/dns_dot.pcap.out | 16 +- test/results/dns_exfiltration.pcap.out | 16 +- test/results/dns_fragmented.pcap.out | 130 +- test/results/dns_invert_query.pcapng.out | 12 +- test/results/dns_long_domainname.pcap.out | 16 +- .../dnscrypt-v1-and-resolver-pings.pcap.out | 1220 +++++++-------- test/results/dnscrypt-v2-doh.pcap.out | 214 +-- test/results/dnscrypt-v2.pcap.out | 32 + .../dnscrypt_skype_false_positive.pcapng.out | 6 +- test/results/doq.pcapng.out | 14 +- test/results/doq_adguard.pcapng.out | 10 +- test/results/dos_win98_smb_netbeui.pcap.out | 30 +- test/results/drda_db2.pcap.out | 14 +- test/results/dropbox.pcap.out | 82 +- test/results/dtls.pcap.out | 12 +- test/results/dtls2.pcap.out | 22 +- test/results/dtls_certificate.pcapng.out | 21 + test/results/dtls_certificate_fragments.pcap.out | 16 +- .../dtls_session_id_and_coockie_both.pcap.out | 16 +- test/results/encrypted_sni.pcap.out | 12 +- test/results/ethereum.pcap.out | 296 ++-- test/results/ethernetIP.pcap.out | 24 +- test/results/exe_download.pcap.out | 16 +- test/results/exe_download_as_png.pcap.out | 16 +- test/results/facebook.pcap.out | 22 +- test/results/firefox.pcap.out | 46 +- test/results/fix.pcap.out | 58 +- test/results/forticlient.pcap.out | 50 +- test/results/ftp-start-tls.pcap.out | 14 +- test/results/ftp.pcap.out | 18 +- test/results/ftp_failed.pcap.out | 12 +- test/results/fuzz-2006-06-26-2594.pcap.out | 1006 ++++++------ test/results/fuzz-2006-09-29-28586.pcap.out | 83 +- test/results/fuzz-2020-02-16-11740.pcap.out | 368 ++--- test/results/fuzz-2021-06-07-c6c72a0a56.pcap.out | 6 +- test/results/fuzz-2021-10-13.pcap.out | 6 +- test/results/genshin-impact.pcap.out | 22 +- test/results/git.pcap.out | 14 +- test/results/google_ssl.pcap.out | 12 +- test/results/googledns_android10.pcap.out | 56 +- test/results/gquic.pcap.out | 10 +- test/results/gtp_false_positive.pcapng.out | 12 +- test/results/h323-overflow.pcap.out | 12 +- test/results/hangout.pcap.out | 14 +- test/results/hpvirtgrp.pcap.out | 48 +- test/results/hsrp0.pcap.out | 33 + test/results/hsrp2.pcap.out | 25 + test/results/hsrp2_ipv6.pcapng.out | 31 + .../http-crash-content-disposition.pcap.out | 6 +- test/results/http-lines-split.pcap.out | 14 +- test/results/http-manipulated.pcap.out | 18 +- test/results/http_auth.pcap.out | 14 +- test/results/http_ipv6.pcap.out | 70 +- test/results/iec60780-5-104.pcap.out | 34 +- test/results/imap-starttls.pcap.out | 14 +- test/results/imap.pcap.out | 14 +- test/results/imaps.pcap.out | 18 +- test/results/instagram.pcap.out | 178 +-- test/results/ip_fragmented_garbage.pcap.out | 6 +- test/results/iphone.pcap.out | 286 ++-- test/results/ipv6_in_gtp.pcap.out | 6 +- test/results/irc.pcap.out | 14 +- test/results/ja3_lots_of_cipher_suites.pcap.out | 6 +- .../ja3_lots_of_cipher_suites_2_anon.pcap.out | 10 +- test/results/kerberos.pcap.out | 92 +- test/results/kerberos_fuzz.pcapng.out | 8 +- test/results/log4j-webapp-exploit.pcap.out | 34 +- test/results/long_tls_certificate.pcap.out | 18 +- test/results/malformed_dns.pcap.out | 12 +- test/results/malformed_icmp.pcap.out | 14 +- test/results/malware.pcap.out | 32 +- test/results/memcached.cap.out | 14 +- test/results/modbus.pcap.out | 14 +- test/results/monero.pcap.out | 18 +- test/results/mongodb.pcap.out | 30 +- test/results/mpeg.pcap.out | 16 +- test/results/mpegts.pcap.out | 10 +- test/results/mqtt.pcap.out | 27 + test/results/mssql_tds.pcap.out | 52 +- test/results/mysql-8.pcap.out | 14 +- test/results/nats.pcap.out | 18 +- ...ndpi_match_string_subprotocol__error.pcapng.out | 12 +- test/results/nest_log_sink.pcap.out | 96 +- test/results/netbios.pcap.out | 68 +- test/results/netbios_wildcard_dns_query.pcap.out | 12 +- test/results/netflix.pcap.out | 347 ++--- test/results/netflow-fritz.pcap.out | 10 +- test/results/netflowv9.pcap.out | 10 +- test/results/nintendo.pcap.out | 98 +- test/results/no_sni.pcap.out | 54 +- test/results/ocs.pcap.out | 6 +- test/results/ocsp.pcapng.out | 50 +- test/results/ookla.pcap.out | 16 +- test/results/openvpn.pcap.out | 22 +- test/results/os_detected.pcapng.out | 10 +- test/results/pinterest.pcap.out | 190 +-- test/results/pop3.pcap.out | 14 +- test/results/pps.pcap.out | 283 ++-- test/results/ps_vue.pcap.out | 60 - test/results/punycode-idn.pcap.out | 35 + test/results/quic-23.pcap.out | 10 +- test/results/quic-24.pcap.out | 10 +- test/results/quic-27.pcap.out | 10 +- test/results/quic-28.pcap.out | 10 +- test/results/quic-29.pcap.out | 10 +- test/results/quic-33.pcapng.out | 10 +- test/results/quic-fuzz-overflow.pcapng.out | 10 +- test/results/quic-mvfst-22.pcap.out | 12 +- .../quic-mvfst-22_decryption_error.pcap.out | 6 +- test/results/quic-mvfst-27.pcapng.out | 10 +- test/results/quic-mvfst-exp.pcap.out | 10 +- test/results/quic-v2-00.pcapng.out | 10 +- test/results/quic.pcap.out | 44 +- test/results/quic046.pcap.out | 10 +- test/results/quic_0RTT.pcap.out | 10 +- .../quic_frags_ch_in_multiple_packets.pcapng.out | 12 +- ...h_out_of_order_same_packet_craziness.pcapng.out | 242 +-- test/results/quic_interop_V.pcapng.out | 314 ++-- test/results/quic_q39.pcap.out | 10 +- test/results/quic_q43.pcap.out | 10 +- test/results/quic_q46.pcap.out | 10 +- test/results/quic_q46_b.pcap.out | 10 +- test/results/quic_q50.pcap.out | 10 +- test/results/quic_t50.pcap.out | 10 +- test/results/quic_t51.pcap.out | 10 +- test/results/quickplay.pcap.out | 74 +- test/results/radius_false_positive.pcapng.out | 6 +- test/results/rdp.pcap.out | 14 +- test/results/reasm_crash_anon.pcapng.out | 6 +- test/results/reasm_segv_anon.pcapng.out | 14 +- test/results/reddit.pcap.out | 416 ++--- test/results/rtsp.pcap.out | 38 +- test/results/rtsp_setup_http.pcapng.out | 14 +- test/results/rx.pcap.out | 26 +- test/results/s7comm.pcap.out | 14 +- test/results/safari.pcap.out | 46 +- test/results/salesforce.pcap.out | 18 +- test/results/selfsigned.pcap.out | 16 +- test/results/signal.pcap.out | 128 +- test/results/simple-dnscrypt.pcap.out | 42 +- test/results/sip.pcap.out | 56 +- test/results/skype-conference-call.pcap.out | 14 +- test/results/skype.pcap.out | 1175 +++++++------- test/results/skype_no_unknown.pcap.out | 853 ++++++----- test/results/skype_udp.pcap.out | 14 +- test/results/smb_deletefile.pcap.out | 10 +- test/results/smbv1.pcap.out | 14 +- test/results/smpp_in_general.pcap.out | 14 +- test/results/smtp-starttls.pcap.out | 14 +- test/results/smtp.pcap.out | 14 +- test/results/snapchat.pcap.out | 22 +- test/results/snapchat_call.pcapng.out | 10 +- test/results/ssdp-m-search.pcap.out | 16 +- test/results/ssh.pcap.out | 23 +- test/results/ssl-cert-name-mismatch.pcap.out | 18 +- test/results/starcraft_battle.pcap.out | 194 +-- test/results/steam.pcap.out | 230 +-- test/results/steam_datagram_relay_ping.pcapng.out | 10 +- test/results/stun_facebook.pcapng.out | 14 +- test/results/stun_signal.pcapng.out | 102 +- test/results/synscan.pcap.out | 250 +-- test/results/syslog.pcapng.out | 40 +- test/results/teams.pcap.out | 402 ++--- test/results/teamspeak3.pcap.out | 12 +- test/results/telegram.pcap.out | 208 +-- test/results/teredo.pcap.out | 30 +- test/results/tftp.pcap.out | 22 +- test/results/tinc.pcap.out | 22 +- test/results/tk.pcap.out | 28 +- test/results/tls-esni-fuzzed.pcap.out | 12 +- test/results/tls-rdn-extract.pcap.out | 18 +- test/results/tls_alert.pcap.out | 12 +- test/results/tls_certificate_too_long.pcap.out | 184 +-- test/results/tls_cipher_lens.pcap.out | 20 +- test/results/tls_esni_sni_both.pcap.out | 22 +- test/results/tls_invalid_reads.pcap.out | 16 +- test/results/tls_long_cert.pcap.out | 18 +- test/results/tls_port_80.pcapng.out | 14 +- test/results/tls_torrent.pcapng.out | 18 +- test/results/tls_verylong_certificate.pcap.out | 18 +- test/results/tor.pcap.out | 76 +- test/results/trickbot.pcap.out | 16 +- test/results/tumblr.pcap.out | 158 +- test/results/ubntac2.pcap.out | 42 +- test/results/upnp.pcap.out | 10 +- test/results/viber.pcap.out | 132 +- test/results/vnc.pcap.out | 18 +- test/results/vxlan.pcap.out | 67 + test/results/wa_video.pcap.out | 64 +- test/results/wa_voice.pcap.out | 126 +- test/results/waze.pcap.out | 173 ++- test/results/webex.pcap.out | 265 ++-- test/results/websocket.pcap.out | 14 +- test/results/wechat.pcap.out | 532 +++---- test/results/weibo.pcap.out | 155 +- test/results/whatsapp_login_call.pcap.out | 236 +-- test/results/whatsapp_login_chat.pcap.out | 44 +- test/results/whatsapp_voice_and_message.pcap.out | 85 +- test/results/whatsappfiles.pcap.out | 22 +- test/results/whois.pcapng.out | 20 +- test/results/wireguard.pcap.out | 18 +- test/results/youtube_quic.pcap.out | 18 +- test/results/youtubeupload.pcap.out | 22 +- test/results/z3950.pcapng.out | 16 +- test/results/zabbix.pcap.out | 14 +- test/results/zattoo.pcap.out | 30 + test/results/zcash.pcap.out | 14 +- test/results/zoom.pcap.out | 170 +-- test/results/zoom2.pcap.out | 40 +- 280 files changed, 11179 insertions(+), 10940 deletions(-) delete mode 100644 test/results/bittorrent_ip.pcap.out create mode 100644 test/results/bot.pcap.out create mode 100644 test/results/dnscrypt-v2.pcap.out create mode 100644 test/results/dtls_certificate.pcapng.out create mode 100644 test/results/hsrp0.pcap.out create mode 100644 test/results/hsrp2.pcap.out create mode 100644 test/results/hsrp2_ipv6.pcapng.out create mode 100644 test/results/mqtt.pcap.out delete mode 100644 test/results/ps_vue.pcap.out create mode 100644 test/results/punycode-idn.pcap.out create mode 100644 test/results/vxlan.pcap.out create mode 100644 test/results/zattoo.pcap.out diff --git a/CMakeLists.txt b/CMakeLists.txt index 4761e2910..9fadb2a95 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -308,8 +308,28 @@ endif() install(TARGETS nDPId DESTINATION sbin) install(TARGETS nDPIsrvd nDPId-test DESTINATION bin) -install(FILES dependencies/nDPIsrvd.py DESTINATION share/nDPId) -install(FILES examples/py-flow-info/flow-info.py DESTINATION bin RENAME nDPIsrvd-flow-info.py) +install(FILES dependencies/nDPIsrvd.py examples/py-flow-dashboard/plotly_dash.py + DESTINATION share/nDPId) +install(FILES examples/py-flow-info/flow-info.py + DESTINATION bin RENAME nDPIsrvd-flow-info.py + PERMISSIONS OWNER_READ OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE) +install(FILES examples/py-flow-dashboard/flow-dash.py + DESTINATION bin RENAME nDPIsrvd-flow-dash.py + PERMISSIONS OWNER_READ OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE) +install(FILES examples/py-ja3-checker/py-ja3-checker.py + DESTINATION bin RENAME nDPIsrvd-ja3-checker.py + PERMISSIONS OWNER_READ OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE) +install(FILES examples/py-json-stdout/json-stdout.py + DESTINATION bin RENAME nDPIsrvd-json-stdout.py + PERMISSIONS OWNER_READ OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE) +install(FILES examples/py-schema-validation/py-schema-validation.py + DESTINATION bin RENAME nDPIsrvd-schema-validation.py + PERMISSIONS OWNER_READ OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE) +install(FILES examples/py-semantic-validation/py-semantic-validation.py + DESTINATION bin RENAME nDPIsrvd-semantic-validation.py + PERMISSIONS OWNER_READ OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE) +install(FILES schema/basic_event_schema.json schema/daemon_event_schema.json + schema/flow_event_schema.json schema/packet_event_schema.json DESTINATION share/nDPId) message(STATUS "--------------------------") message(STATUS "nDPId GIT_VERSION........: ${GIT_VERSION}") diff --git a/dependencies/nDPIsrvd.h b/dependencies/nDPIsrvd.h index 535941aa1..a4d077234 100644 --- a/dependencies/nDPIsrvd.h +++ b/dependencies/nDPIsrvd.h @@ -25,7 +25,7 @@ #include #endif -#define nDPIsrvd_MAX_JSON_TOKENS 128 +#define nDPIsrvd_MAX_JSON_TOKENS 256 #define nDPIsrvd_JSON_KEY_STRLEN 32 #define nDPIsrvd_STRLEN_SZ(s) (sizeof(s) / sizeof(s[0]) - sizeof(s[0])) @@ -67,7 +67,11 @@ enum nDPIsrvd_parse_return PARSE_SIZE_MISSING, PARSE_STRING_TOO_BIG, PARSE_INVALID_CLOSING_CHAR, - PARSE_JSMN_ERROR, + PARSE_JSMN_KEY_MISSING, + PARSE_JSMN_NOMEM, + PARSE_JSMN_INVALID, + PARSE_JSMN_PARTIAL, + PARSE_JSMN_UNKNOWN_ERROR, PARSE_JSON_CALLBACK_ERROR, PARSE_JSON_MGMT_ERROR, PARSE_FLOW_MGMT_ERROR, @@ -306,7 +310,11 @@ static inline char const * nDPIsrvd_enum_to_string(int enum_value) "PARSE_SIZE_MISSING", "PARSE_STRING_TOO_BIG", "PARSE_INVALID_CLOSING_CHAR", - "PARSE_JSMN_ERROR", + "PARSE_JSMN_KEY_MISSING", + "PARSE_JSMN_NOMEM", + "PARSE_JSMN_INVALID", + "PARSE_JSMN_PARTIAL", + "PARSE_JSMN_UNKNOWN_ERROR", "PARSE_JSON_CALLBACK_ERROR", "PARSE_JSON_MGMT_ERROR", "PARSE_FLOW_MGMT_ERROR", @@ -569,6 +577,11 @@ static inline enum nDPIsrvd_connect_return nDPIsrvd_connect(struct nDPIsrvd_sock static inline enum nDPIsrvd_read_return nDPIsrvd_read(struct nDPIsrvd_socket * const sock) { + if (sock->buffer.used == sock->buffer.max) + { + return READ_OK; + } + ssize_t bytes_read = read(sock->fd, sock->buffer.ptr.raw + sock->buffer.used, sock->buffer.max - sock->buffer.used); @@ -966,7 +979,17 @@ static inline enum nDPIsrvd_parse_return nDPIsrvd_parse_line(struct nDPIsrvd_buf nDPIsrvd_MAX_JSON_TOKENS); if (jsmn->tokens_found < 0 || jsmn->tokens[0].type != JSMN_OBJECT) { - return PARSE_JSMN_ERROR; + switch ((enum jsmnerr)jsmn->tokens_found) + { + case JSMN_ERROR_NOMEM: + return PARSE_JSMN_NOMEM; + case JSMN_ERROR_INVAL: + return PARSE_JSMN_INVALID; + case JSMN_ERROR_PART: + return PARSE_JSMN_PARTIAL; + } + + return PARSE_JSMN_UNKNOWN_ERROR; } return PARSE_OK; @@ -994,7 +1017,7 @@ static inline enum nDPIsrvd_parse_return nDPIsrvd_parse_all(struct nDPIsrvd_sock { if (key != NULL) { - ret = PARSE_JSMN_ERROR; + ret = PARSE_JSMN_KEY_MISSING; break; } @@ -1003,7 +1026,7 @@ static inline enum nDPIsrvd_parse_return nDPIsrvd_parse_all(struct nDPIsrvd_sock if (key == NULL) { - ret = PARSE_JSMN_ERROR; + ret = PARSE_JSMN_KEY_MISSING; break; } } diff --git a/dependencies/nDPIsrvd.py b/dependencies/nDPIsrvd.py index 1d0dec872..0543f1a24 100644 --- a/dependencies/nDPIsrvd.py +++ b/dependencies/nDPIsrvd.py @@ -427,10 +427,21 @@ def validateAddress(args): global schema schema = {'packet_event_schema' : None, 'basic_event_schema' : None, 'daemon_event_schema' : None, 'flow_event_schema' : None} -def initSchemaValidator(schema_dir='./schema'): +def initSchemaValidator(schema_dirs=[]): + if len(schema_dirs) == 0: + schema_dirs += [os.path.dirname(sys.argv[0]) + '/../../schema'] + schema_dirs += [os.path.dirname(sys.argv[0]) + '/../share/nDPId'] + schema_dirs += [sys.base_prefix + '/share/nDPId'] + for key in schema: - with open(schema_dir + '/' + str(key) + '.json', 'r') as schema_file: - schema[key] = json.load(schema_file) + for schema_dir in schema_dirs: + try: + with open(schema_dir + '/' + str(key) + '.json', 'r') as schema_file: + schema[key] = json.load(schema_file) + except FileNotFoundError: + continue + else: + break def validateAgainstSchema(json_dict): import jsonschema diff --git a/examples/py-flow-dashboard/flow-dash.py b/examples/py-flow-dashboard/flow-dash.py index 947f5b48c..411029398 100755 --- a/examples/py-flow-dashboard/flow-dash.py +++ b/examples/py-flow-dashboard/flow-dash.py @@ -4,18 +4,12 @@ import multiprocessing import os import sys -import plotly_dash - +sys.path.append(os.path.dirname(sys.argv[0]) + '/../../dependencies') sys.path.append(os.path.dirname(sys.argv[0]) + '/../share/nDPId') -sys.path.append(os.path.dirname(sys.argv[0]) + '/../usr/share/nDPId') -try: - import nDPIsrvd - from nDPIsrvd import nDPIsrvdSocket -except ImportError: - sys.path.append(os.path.dirname(sys.argv[0]) + '/../../dependencies') - import nDPIsrvd - from nDPIsrvd import nDPIsrvdSocket - +sys.path.append(sys.base_prefix + '/share/nDPId') +import nDPIsrvd +from nDPIsrvd import nDPIsrvdSocket +import plotly_dash def nDPIsrvd_worker_onFlowCleanup(instance, current_flow, global_user_data): _, shared_flow_dict = global_user_data diff --git a/examples/py-flow-info/flow-info.py b/examples/py-flow-info/flow-info.py index bb1326777..06bbf83fb 100755 --- a/examples/py-flow-info/flow-info.py +++ b/examples/py-flow-info/flow-info.py @@ -6,15 +6,11 @@ import sys import time import datetime +sys.path.append(os.path.dirname(sys.argv[0]) + '/../../dependencies') sys.path.append(os.path.dirname(sys.argv[0]) + '/../share/nDPId') -sys.path.append(os.path.dirname(sys.argv[0]) + '/../usr/share/nDPId') -try: - import nDPIsrvd - from nDPIsrvd import nDPIsrvdSocket, TermColor -except ImportError: - sys.path.append(os.path.dirname(sys.argv[0]) + '/../../dependencies') - import nDPIsrvd - from nDPIsrvd import nDPIsrvdSocket, TermColor +sys.path.append(sys.base_prefix + '/share/nDPId') +import nDPIsrvd +from nDPIsrvd import nDPIsrvdSocket, TermColor global args global whois_db diff --git a/examples/py-flow-muliprocess/py-flow-multiprocess.py b/examples/py-flow-muliprocess/py-flow-multiprocess.py index b90ab536d..9014c5f59 100755 --- a/examples/py-flow-muliprocess/py-flow-multiprocess.py +++ b/examples/py-flow-muliprocess/py-flow-multiprocess.py @@ -4,16 +4,11 @@ import multiprocessing import os import sys +sys.path.append(os.path.dirname(sys.argv[0]) + '/../../dependencies') sys.path.append(os.path.dirname(sys.argv[0]) + '/../share/nDPId') sys.path.append(os.path.dirname(sys.argv[0]) + '/../usr/share/nDPId') -try: - import nDPIsrvd - from nDPIsrvd import nDPIsrvdSocket -except ImportError: - sys.path.append(os.path.dirname(sys.argv[0]) + '/../../dependencies') - import nDPIsrvd - from nDPIsrvd import nDPIsrvdSocket - +import nDPIsrvd +from nDPIsrvd import nDPIsrvdSocket def mp_worker(unused, shared_flow_dict): import time diff --git a/examples/py-ja3-checker/py-ja3-checker.py b/examples/py-ja3-checker/py-ja3-checker.py index b7f9df5b1..d3db3b56e 100755 --- a/examples/py-ja3-checker/py-ja3-checker.py +++ b/examples/py-ja3-checker/py-ja3-checker.py @@ -8,15 +8,11 @@ import requests import sys import time +sys.path.append(os.path.dirname(sys.argv[0]) + '/../../dependencies') sys.path.append(os.path.dirname(sys.argv[0]) + '/../share/nDPId') sys.path.append(os.path.dirname(sys.argv[0]) + '/../usr/share/nDPId') -try: - import nDPIsrvd - from nDPIsrvd import nDPIsrvdSocket -except ImportError: - sys.path.append(os.path.dirname(sys.argv[0]) + '/../../dependencies') - import nDPIsrvd - from nDPIsrvd import nDPIsrvdSocket +import nDPIsrvd +from nDPIsrvd import nDPIsrvdSocket global ja3_fps ja3_fps = dict() diff --git a/examples/py-json-stdout/json-stdout.py b/examples/py-json-stdout/json-stdout.py index 160a61e0c..bd27d7d80 100755 --- a/examples/py-json-stdout/json-stdout.py +++ b/examples/py-json-stdout/json-stdout.py @@ -3,15 +3,11 @@ import os import sys +sys.path.append(os.path.dirname(sys.argv[0]) + '/../../dependencies') sys.path.append(os.path.dirname(sys.argv[0]) + '/../share/nDPId') sys.path.append(os.path.dirname(sys.argv[0]) + '/../usr/share/nDPId') -try: - import nDPIsrvd - from nDPIsrvd import nDPIsrvdSocket, TermColor -except ImportError: - sys.path.append(os.path.dirname(sys.argv[0]) + '/../../dependencies') - import nDPIsrvd - from nDPIsrvd import nDPIsrvdSocket, TermColor +import nDPIsrvd +from nDPIsrvd import nDPIsrvdSocket, TermColor def onJsonLineRecvd(json_dict, instance, current_flow, global_user_data): print(json_dict) diff --git a/examples/py-schema-validation/py-schema-validation.py b/examples/py-schema-validation/py-schema-validation.py index 6a07681b3..590ace92e 100755 --- a/examples/py-schema-validation/py-schema-validation.py +++ b/examples/py-schema-validation/py-schema-validation.py @@ -3,15 +3,11 @@ import os import sys +sys.path.append(os.path.dirname(sys.argv[0]) + '/../../dependencies') sys.path.append(os.path.dirname(sys.argv[0]) + '/../share/nDPId') sys.path.append(os.path.dirname(sys.argv[0]) + '/../usr/share/nDPId') -try: - import nDPIsrvd - from nDPIsrvd import nDPIsrvdSocket, TermColor -except ImportError: - sys.path.append(os.path.dirname(sys.argv[0]) + '/../../dependencies') - import nDPIsrvd - from nDPIsrvd import nDPIsrvdSocket, TermColor +import nDPIsrvd +from nDPIsrvd import nDPIsrvdSocket, TermColor class Stats: lines_processed = 0 @@ -40,7 +36,7 @@ if __name__ == '__main__': sys.stderr.write('Recv buffer size: {}\n'.format(nDPIsrvd.NETWORK_BUFFER_MAX_SIZE)) sys.stderr.write('Connecting to {} ..\n'.format(address[0]+':'+str(address[1]) if type(address) is tuple else address)) - nDPIsrvd.initSchemaValidator(os.path.dirname(sys.argv[0]) + '/../../schema') + nDPIsrvd.initSchemaValidator() nsock = nDPIsrvdSocket() nsock.connect(address) diff --git a/examples/py-semantic-validation/py-semantic-validation.py b/examples/py-semantic-validation/py-semantic-validation.py index 019194dac..3ca90edcf 100755 --- a/examples/py-semantic-validation/py-semantic-validation.py +++ b/examples/py-semantic-validation/py-semantic-validation.py @@ -3,16 +3,11 @@ import os import sys +sys.path.append(os.path.dirname(sys.argv[0]) + '/../../dependencies') sys.path.append(os.path.dirname(sys.argv[0]) + '/../share/nDPId') sys.path.append(os.path.dirname(sys.argv[0]) + '/../usr/share/nDPId') -try: - import nDPIsrvd - from nDPIsrvd import nDPIsrvdSocket, TermColor -except ImportError: - sys.path.append(os.path.dirname(sys.argv[0]) + '/../../dependencies') - import nDPIsrvd - from nDPIsrvd import nDPIsrvdSocket, TermColor - +import nDPIsrvd +from nDPIsrvd import nDPIsrvdSocket, TermColor class Stats: event_counter = dict() diff --git a/libnDPI b/libnDPI index c53c82d48..8b062295c 160000 --- a/libnDPI +++ b/libnDPI @@ -1 +1 @@ -Subproject commit c53c82d4823b5a8f856d1375155ac5112b68e8af +Subproject commit 8b062295cc76a60e3905c054ce37bd17669464d1 diff --git a/nDPId-test.c b/nDPId-test.c index 9300420ea..b2bad7673 100644 --- a/nDPId-test.c +++ b/nDPId-test.c @@ -548,6 +548,12 @@ static void * distributor_client_mainloop_thread(void * const arg) if (parse_ret != PARSE_NEED_MORE_DATA) { logger(1, "JSON parsing failed: %s", nDPIsrvd_enum_to_string(parse_ret)); + logger(1, "Problematic JSON string (start: %zu, length: %llu, buffer usage: %zu): %.*s", + mock_sock->buffer.json_string_start, + mock_sock->buffer.json_string_length, + mock_sock->buffer.used, + (int)mock_sock->buffer.json_string_length, + mock_sock->buffer.json_string); THREAD_ERROR_GOTO(trv); } diff --git a/nDPId.c b/nDPId.c index 53929aed4..682f0bae0 100644 --- a/nDPId.c +++ b/nDPId.c @@ -122,6 +122,7 @@ struct nDPId_flow_extended uint64_t last_flow_update; unsigned long long int total_l4_payload_len; + struct ndpi_proto detected_l7_protocol; }; /* @@ -139,11 +140,8 @@ struct nDPId_flow_skipped struct nDPId_detection_data { uint32_t last_ndpi_flow_struct_hash; - struct ndpi_proto detected_l7_protocol; struct ndpi_proto guessed_l7_protocol; struct ndpi_flow_struct flow; - struct ndpi_id_struct src; - struct ndpi_id_struct dst; }; struct nDPId_flow_info @@ -151,8 +149,7 @@ struct nDPId_flow_info struct nDPId_flow_extended flow_extended; uint8_t detection_completed : 1; - uint8_t extra_dissection_completed : 1; - uint8_t reserved_00 : 6; + uint8_t reserved_00 : 7; uint8_t reserved_01[1]; #ifdef ENABLE_ZLIB uint16_t detection_data_compressed_size; @@ -167,8 +164,8 @@ struct nDPId_flow_finished { struct nDPId_flow_extended flow_extended; - struct ndpi_proto detected_l7_protocol; ndpi_risk risk; + ndpi_confidence_t confidence; }; /* TODO: Merge `struct nDPId_flow_info' with `struct nDPId_flow_finished' and use a union instead? */ @@ -621,15 +618,6 @@ static int detection_data_inflate(struct nDPId_flow_info * const flow_info) memcpy(flow_info->detection_data, tmpOut, ret); flow_info->detection_data_compressed_size = 0; - /* - * The ndpi_id_struct's from ndpi_flow may not be valid anymore. - * nDPI only updates those pointers while processing packets! - * This is especially important when using compression - * to prevent use of dangling pointers. - */ - flow_info->detection_data->flow.src = &flow_info->detection_data->src; - flow_info->detection_data->flow.dst = &flow_info->detection_data->dst; - return ret; } @@ -2381,7 +2369,8 @@ static void jsonize_flow_event(struct nDPId_reader_thread * const reader_thread, ndpi_serialize_proto(workflow->ndpi_struct, &workflow->ndpi_serializer, flow_finished->risk, - flow_finished->detected_l7_protocol); + flow_finished->confidence, + flow_finished->flow_extended.detected_l7_protocol); } break; @@ -2453,7 +2442,7 @@ static void jsonize_flow_detection_event(struct nDPId_reader_thread * const read case FLOW_EVENT_DETECTION_UPDATE: if (ndpi_dpi2json(workflow->ndpi_struct, &flow_info->detection_data->flow, - flow_info->detection_data->detected_l7_protocol, + flow_info->flow_extended.detected_l7_protocol, &workflow->ndpi_serializer) != 0) { logger(1, @@ -2704,13 +2693,14 @@ static uint32_t calculate_ndpi_flow_struct_hash(struct ndpi_flow_struct const * uint32_t hash = murmur3_32((uint8_t const *)&ndpi_flow->protos, sizeof(ndpi_flow->protos), nDPId_FLOW_STRUCT_SEED); hash += ndpi_flow->category; hash += ndpi_flow->risk; + hash += ndpi_flow->confidence; - const size_t protocol_bitmask_size = sizeof(ndpi_flow->src->detected_protocol_bitmask.fds_bits) / - sizeof(ndpi_flow->src->detected_protocol_bitmask.fds_bits[0]); + const size_t protocol_bitmask_size = sizeof(ndpi_flow->excluded_protocol_bitmask.fds_bits) / + sizeof(ndpi_flow->excluded_protocol_bitmask.fds_bits[0]); for (size_t i = 0; i < protocol_bitmask_size; ++i) { - hash += ndpi_flow->src->detected_protocol_bitmask.fds_bits[i]; - hash += ndpi_flow->dst->detected_protocol_bitmask.fds_bits[i]; + hash += ndpi_flow->excluded_protocol_bitmask.fds_bits[i]; + hash += ndpi_flow->excluded_protocol_bitmask.fds_bits[i]; } size_t host_server_name_len = @@ -3106,10 +3096,7 @@ static void ndpi_process_packet(uint8_t * const args, void * tree_result; struct nDPId_flow_info * flow_to_process; - uint8_t direction_changed = 0; uint8_t is_new_flow = 0; - struct ndpi_id_struct * ndpi_src; - struct ndpi_id_struct * ndpi_dst; const struct ndpi_iphdr * ip; struct ndpi_ipv6hdr * ip6; @@ -3376,10 +3363,6 @@ static void ndpi_process_packet(uint8_t * const args, flow_basic.dst_port = orig_src_port; tree_result = ndpi_tfind(&flow_basic, &workflow->ndpi_flows_active[hashed_index], ndpi_workflow_node_cmp); - if (tree_result != NULL) - { - direction_changed = 1; - } flow_basic.src.v6.ip[0] = orig_src_ip[0]; flow_basic.src.v6.ip[1] = orig_src_ip[1]; @@ -3496,9 +3479,6 @@ static void ndpi_process_packet(uint8_t * const args, return; } - ndpi_src = &flow_to_process->detection_data->src; - ndpi_dst = &flow_to_process->detection_data->dst; - is_new_flow = 1; } else @@ -3528,9 +3508,6 @@ static void ndpi_process_packet(uint8_t * const args, } flow_to_process = (struct nDPId_flow_info *)flow_basic_to_process; - ndpi_src = NULL; - ndpi_dst = NULL; - if (flow_to_process->flow_extended.flow_basic.state == FS_INFO) { #ifdef ENABLE_ZLIB @@ -3547,20 +3524,6 @@ static void ndpi_process_packet(uint8_t * const args, } } #endif - - if (flow_to_process->detection_data != NULL) - { - if (direction_changed != 0) - { - ndpi_src = &flow_to_process->detection_data->dst; - ndpi_dst = &flow_to_process->detection_data->src; - } - else - { - ndpi_src = &flow_to_process->detection_data->src; - ndpi_dst = &flow_to_process->detection_data->dst; - } - } } } @@ -3632,16 +3595,14 @@ static void ndpi_process_packet(uint8_t * const args, } } - flow_to_process->detection_data->detected_l7_protocol = + flow_to_process->flow_extended.detected_l7_protocol = ndpi_detection_process_packet(workflow->ndpi_struct, &flow_to_process->detection_data->flow, ip != NULL ? (uint8_t *)ip : (uint8_t *)ip6, ip_size, - workflow->last_time, - ndpi_src, - ndpi_dst); + workflow->last_time); - if (ndpi_is_protocol_detected(workflow->ndpi_struct, flow_to_process->detection_data->detected_l7_protocol) != 0 && + if (ndpi_is_protocol_detected(workflow->ndpi_struct, flow_to_process->flow_extended.detected_l7_protocol) != 0 && flow_to_process->detection_completed == 0) { flow_to_process->detection_completed = 1; @@ -3649,13 +3610,8 @@ static void ndpi_process_packet(uint8_t * const args, jsonize_flow_detection_event(reader_thread, flow_to_process, FLOW_EVENT_DETECTED); flow_to_process->detection_data->last_ndpi_flow_struct_hash = calculate_ndpi_flow_struct_hash(&flow_to_process->detection_data->flow); - - if (ndpi_extra_dissection_possible(workflow->ndpi_struct, &flow_to_process->detection_data->flow) == 0) - { - flow_to_process->extra_dissection_completed = 1; - } } - else if (flow_to_process->detection_completed == 1 && flow_to_process->extra_dissection_completed == 0) + else if (flow_to_process->detection_completed == 1) { uint32_t hash = calculate_ndpi_flow_struct_hash(&flow_to_process->detection_data->flow); if (hash != flow_to_process->detection_data->last_ndpi_flow_struct_hash) @@ -3664,34 +3620,28 @@ static void ndpi_process_packet(uint8_t * const args, jsonize_flow_detection_event(reader_thread, flow_to_process, FLOW_EVENT_DETECTION_UPDATE); flow_to_process->detection_data->last_ndpi_flow_struct_hash = hash; } - - if (ndpi_extra_dissection_possible(workflow->ndpi_struct, &flow_to_process->detection_data->flow) == 0) - { - flow_to_process->extra_dissection_completed = 1; - } } if (flow_to_process->detection_data->flow.num_processed_pkts == nDPId_options.max_packets_per_flow_to_process || - (flow_to_process->detection_completed == 1 && flow_to_process->extra_dissection_completed == 1)) + (flow_to_process->detection_completed == 1 && + ndpi_extra_dissection_possible(workflow->ndpi_struct, &flow_to_process->detection_data->flow) == 0)) { - struct ndpi_proto detected_l7_protocol = flow_to_process->detection_data->detected_l7_protocol; + struct ndpi_proto detected_l7_protocol = flow_to_process->flow_extended.detected_l7_protocol; + if (ndpi_is_protocol_detected(workflow->ndpi_struct, detected_l7_protocol) == 0) + { + detected_l7_protocol = flow_to_process->detection_data->guessed_l7_protocol; + } + ndpi_risk risk = flow_to_process->detection_data->flow.risk; + ndpi_confidence_t confidence = flow_to_process->detection_data->flow.confidence; free_detection_data(flow_to_process); flow_to_process->flow_extended.flow_basic.state = FS_FINISHED; struct nDPId_flow_finished * const flow_finished = (struct nDPId_flow_finished *)flow_to_process; - if (flow_to_process->detection_completed == 0) - { - struct ndpi_proto unknown_l7_protocol = {}; - flow_finished->detected_l7_protocol = unknown_l7_protocol; - flow_finished->risk = NDPI_NO_RISK; - } - else - { - flow_finished->detected_l7_protocol = detected_l7_protocol; - flow_finished->risk = risk; - } + flow_finished->flow_extended.detected_l7_protocol = detected_l7_protocol; + flow_finished->risk = risk; + flow_finished->confidence = confidence; } #ifdef ENABLE_ZLIB @@ -3854,7 +3804,8 @@ static int start_reader_threads(void) errno = 0; if (nDPId_options.user != NULL && - change_user_group(nDPId_options.user, nDPId_options.group, nDPId_options.pidfile, NULL, NULL) != 0) + change_user_group(nDPId_options.user, nDPId_options.group, nDPId_options.pidfile, NULL, NULL) != 0 && + errno != EPERM) { if (errno != 0) { diff --git a/scripts/get-and-build-libndpi.sh b/scripts/get-and-build-libndpi.sh index 215c7b9b0..f4bae262e 100755 --- a/scripts/get-and-build-libndpi.sh +++ b/scripts/get-and-build-libndpi.sh @@ -10,6 +10,17 @@ flock -x -n 42 || { exit 1; } +cat <