From b3b8feabf1a60bcf9e6e8fc748a0a7a6cd8923f2 Mon Sep 17 00:00:00 2001 From: Darryl Sokoloski Date: Wed, 15 Aug 2018 16:47:21 -0400 Subject: Added new dissector for Memcached. Signed-off-by: Darryl Sokoloski --- src/include/ndpi_protocols.h | 2 ++ 1 file changed, 2 insertions(+) (limited to 'src/include/ndpi_protocols.h') diff --git a/src/include/ndpi_protocols.h b/src/include/ndpi_protocols.h index f08b1de73..5c08f85f5 100644 --- a/src/include/ndpi_protocols.h +++ b/src/include/ndpi_protocols.h @@ -203,6 +203,7 @@ void ndpi_search_tinc(struct ndpi_detection_module_struct *ndpi_struct, struct n void ndpi_search_fix(struct ndpi_detection_module_struct *ndpi_struct, struct ndpi_flow_struct *flow); void ndpi_search_csgo(struct ndpi_detection_module_struct *ndpi_struct, struct ndpi_flow_struct *flow); void ndpi_search_ajp(struct ndpi_detection_module_struct *ndpi_struct, struct ndpi_flow_struct *flow); +void ndpi_search_memcached(struct ndpi_detection_module_struct *ndpi_struct, struct ndpi_flow_struct *flow); /* --- INIT FUNCTIONS --- */ void init_diameter_dissector(struct ndpi_detection_module_struct *ndpi_struct, u_int32_t *id, NDPI_PROTOCOL_BITMASK *detection_bitmask); void init_afp_dissector(struct ndpi_detection_module_struct *ndpi_struct, u_int32_t *id, NDPI_PROTOCOL_BITMASK *detection_bitmask); @@ -356,4 +357,5 @@ void init_apple_push_dissector(struct ndpi_detection_module_struct *ndpi_struct, void init_whatsapp_dissector(struct ndpi_detection_module_struct *ndpi_struct, u_int32_t *id, NDPI_PROTOCOL_BITMASK *detection_bitmask); void init_ajp_dissector(struct ndpi_detection_module_struct *ndpi_struct, u_int32_t *id, NDPI_PROTOCOL_BITMASK *detection_bitmask); void init_fbzero_dissector(struct ndpi_detection_module_struct *ndpi_struct, u_int32_t *id, NDPI_PROTOCOL_BITMASK *detection_bitmask); +void init_memcached_dissector(struct ndpi_detection_module_struct *ndpi_struct, u_int32_t *id, NDPI_PROTOCOL_BITMASK *detection_bitmask); #endif /* __NDPI_PROTOCOLS_H__ */ -- cgit v1.2.3 From c6b427c2521c0916866f932ea1db43334a01b2f4 Mon Sep 17 00:00:00 2001 From: Luca Date: Tue, 18 Sep 2018 18:13:04 +0200 Subject: Implementation of Bitcoin, Ethereum, ZCash, Monero dissectors all identified as mining Implements #503 --- src/include/ndpi_protocol_ids.h | 4 +- src/include/ndpi_protocols.h | 1 + src/lib/ndpi_main.c | 9 ++-- src/lib/protocols/mining.c | 94 ++++++++++++++++++++++++++++++++++++++++ tests/pcap/bitcoin.pcap | Bin 0 -> 591290 bytes tests/pcap/ethereum.pcap | Bin 0 -> 147293 bytes tests/pcap/monero.pcap | Bin 0 -> 171804 bytes tests/pcap/zcash.pcap | Bin 0 -> 22988 bytes tests/result/bitcoin.pcap.out | 8 ++++ tests/result/ethereum.pcap.out | 4 ++ tests/result/monero.pcap.out | 4 ++ tests/result/zcash.pcap.out | 3 ++ 12 files changed, 122 insertions(+), 5 deletions(-) create mode 100644 src/lib/protocols/mining.c create mode 100644 tests/pcap/bitcoin.pcap create mode 100644 tests/pcap/ethereum.pcap create mode 100644 tests/pcap/monero.pcap create mode 100644 tests/pcap/zcash.pcap create mode 100644 tests/result/bitcoin.pcap.out create mode 100644 tests/result/ethereum.pcap.out create mode 100644 tests/result/monero.pcap.out create mode 100644 tests/result/zcash.pcap.out (limited to 'src/include/ndpi_protocols.h') diff --git a/src/include/ndpi_protocol_ids.h b/src/include/ndpi_protocol_ids.h index 7b4961a0e..607e8deef 100644 --- a/src/include/ndpi_protocol_ids.h +++ b/src/include/ndpi_protocol_ids.h @@ -75,9 +75,9 @@ typedef enum { NDPI_PROTOCOL_SKYPE_CALL_OUT = 38, NDPI_PROTOCOL_MUSICALLY = 39, NDPI_PROTOCOL_MEMCACHED = 40, /* Memcached - Darryl Sokoloski */ - NDPI_PROTOCOL_SMBV23 = 41, /* SMB version 2/3 */ + NDPI_PROTOCOL_SMBV23 = 41, /* SMB version 2/3 */ + NDPI_PROTOCOL_MINING = 42, /* Bitcoin, Ethereum, ZCash, Monero */ - NDPI_PROTOCOL_FREE_42 = 42, /* Free */ NDPI_PROTOCOL_FREE_43 = 43, /* Free */ NDPI_PROTOCOL_FREE_44 = 44, /* Free */ NDPI_PROTOCOL_FREE_45 = 45, /* Free */ diff --git a/src/include/ndpi_protocols.h b/src/include/ndpi_protocols.h index 5c08f85f5..e1f3af4fb 100644 --- a/src/include/ndpi_protocols.h +++ b/src/include/ndpi_protocols.h @@ -260,6 +260,7 @@ void init_maplestory_dissector(struct ndpi_detection_module_struct *ndpi_struct, void init_mdns_dissector(struct ndpi_detection_module_struct *ndpi_struct, u_int32_t *id, NDPI_PROTOCOL_BITMASK *detection_bitmask); void init_megaco_dissector(struct ndpi_detection_module_struct *ndpi_struct, u_int32_t *id, NDPI_PROTOCOL_BITMASK *detection_bitmask); void init_mgpc_dissector(struct ndpi_detection_module_struct *ndpi_struct, u_int32_t *id, NDPI_PROTOCOL_BITMASK *detection_bitmask); +void init_mining_dissector(struct ndpi_detection_module_struct *ndpi_struct, u_int32_t *id, NDPI_PROTOCOL_BITMASK *detection_bitmask); void init_mms_dissector(struct ndpi_detection_module_struct *ndpi_struct, u_int32_t *id, NDPI_PROTOCOL_BITMASK *detection_bitmask); void init_msn_dissector(struct ndpi_detection_module_struct *ndpi_struct, u_int32_t *id, NDPI_PROTOCOL_BITMASK *detection_bitmask); void init_mpegts_dissector(struct ndpi_detection_module_struct *ndpi_struct, u_int32_t *id, NDPI_PROTOCOL_BITMASK *detection_bitmask); diff --git a/src/lib/ndpi_main.c b/src/lib/ndpi_main.c index 8fed1ebf1..9b6501d9b 100644 --- a/src/lib/ndpi_main.c +++ b/src/lib/ndpi_main.c @@ -1222,10 +1222,10 @@ static void ndpi_init_protocol_defaults(struct ndpi_detection_module_struct *ndp no_master, "SMBv23", NDPI_PROTOCOL_CATEGORY_SYSTEM_OS, ndpi_build_default_ports(ports_a, 445, 0, 0, 0, 0) /* TCP */, ndpi_build_default_ports(ports_b, 0, 0, 0, 0, 0) /* UDP */); - ndpi_set_proto_defaults(ndpi_mod, NDPI_PROTOCOL_FUN, NDPI_PROTOCOL_FREE_42, + ndpi_set_proto_defaults(ndpi_mod, NDPI_PROTOCOL_FUN, NDPI_PROTOCOL_MINING, no_master, - no_master, "Free", NDPI_PROTOCOL_CATEGORY_CUSTOM_1 /* dummy */, - ndpi_build_default_ports(ports_a, 0, 0, 0, 0, 0) /* TCP */, + no_master, "Mining", CUSTOM_CATEGORY_MINING /* dummy */, + ndpi_build_default_ports(ports_a, 8333, 0, 0, 0, 0) /* TCP */, ndpi_build_default_ports(ports_b, 0, 0, 0, 0, 0) /* UDP */); ndpi_set_proto_defaults(ndpi_mod, NDPI_PROTOCOL_FUN, NDPI_PROTOCOL_FREE_43, no_master, @@ -2929,6 +2929,9 @@ void ndpi_set_protocol_detection_bitmask2(struct ndpi_detection_module_struct *n /* SMB */ init_smb_dissector(ndpi_struct, &a, detection_bitmask); + /* MINING */ + init_mining_dissector(ndpi_struct, &a, detection_bitmask); + /* TELNET */ init_telnet_dissector(ndpi_struct, &a, detection_bitmask); diff --git a/src/lib/protocols/mining.c b/src/lib/protocols/mining.c new file mode 100644 index 000000000..ec094e7d3 --- /dev/null +++ b/src/lib/protocols/mining.c @@ -0,0 +1,94 @@ +/* + * mining.c [Bitcoin, Ethereum, ZCash, Monero] + * + * Copyright (C) 2018 - ntop.org + * + * This file is part of nDPI, an open source deep packet inspection + * library based on the OpenDPI and PACE technology by ipoque GmbH + * + * nDPI is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * nDPI is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with nDPI. If not, see . + * + */ +#include "ndpi_protocol_ids.h" + +#include "ndpi_api.h" + +void ndpi_search_mining_tcp(struct ndpi_detection_module_struct *ndpi_struct, + struct ndpi_flow_struct *flow) { + struct ndpi_packet_struct *packet = &flow->packet; + + NDPI_LOG_DBG(ndpi_struct, "search MINING\n"); + + /* Check connection over TCP */ + if(packet->tcp && (packet->payload_packet_len > 10)) { + + if(packet->tcp->source == htons(8333)) { + /* + Bitcoin + + bitcoin.magic == 0xf9beb4d9 || bitcoin.magic == 0xfabfb5da + */ + u_int32_t magic = htonl(0xf9beb4d9), magic1 = htonl(0xfabfb5da), *to_match = (u_int32_t*)packet->payload; + + if((*to_match == magic) || (*to_match == magic1)) { + ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_MINING, NDPI_PROTOCOL_UNKNOWN); + } + } if(ndpi_strnstr((const char *)packet->payload, "\"eth1.0\"", packet->payload_packet_len) + || ndpi_strnstr((const char *)packet->payload, "\"worker\"", packet->payload_packet_len) + || ndpi_strnstr((const char *)packet->payload, "\"id\"", packet->payload_packet_len)) { + /* + Ethereum + + {"worker": "eth1.0", "jsonrpc": "2.0", "params": ["0x0fccfff9e61a230ff380530c6827caf4759337c6.rig2", "x"], "id": 2, "method": "eth_submitLogin"} + { "id": 2, "jsonrpc":"2.0","result":true} + {"worker": "", "jsonrpc": "2.0", "params": [], "id": 3, "method": "eth_getWork"} + */ + ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_MINING, NDPI_PROTOCOL_UNKNOWN); + } else if(ndpi_strnstr((const char *)packet->payload, "\"method\"", packet->payload_packet_len) + || ndpi_strnstr((const char *)packet->payload, "\"blob\"", packet->payload_packet_len) + || ndpi_strnstr((const char *)packet->payload, "\"id\"", packet->payload_packet_len)) { + /* + ZCash + + {"method":"login","params":{"login":"4BCeEPhodgPMbPWFN1dPwhWXdRX8q4mhhdZdA1dtSMLTLCEYvAj9QXjXAfF7CugEbmfBhgkqHbdgK9b2wKA6nqRZQCgvCDm.cb2b73415c4faf214035a73b9d947c202342f3bf3bdf632132bd6d7af98cb257.ryzen","pass":"x","agent":"xmr-stak-cpu/1.3.0-1.5.0"},"id":1} + {"id":1,"jsonrpc":"2.0","error":null,"result":{"id":"479059546883218","job":{"blob":"0606e89883d205a65d8ee78991838a1cf3ec2ebbc5fb1fa43dec5fa1cd2bee4069212a549cd731000000005a88235653097aa3e97ef2ceef4aee610751a828f9be1a0758a78365fb0a4c8c05","job_id":"722134174127131","target":"dc460300"},"status":"OK"}} + {"method":"submit","params":{"id":"479059546883218","job_id":"722134174127131","nonce":"98024001","result":"c9be9381a68d533c059d614d961e0534d7d8785dd5c339c2f9596eb95f320100"},"id":1} + + Monero + + {"method":"login","params":{"login":"4BCeEPhodgPMbPWFN1dPwhWXdRX8q4mhhdZdA1dtSMLTLCEYvAj9QXjXAfF7CugEbmfBhgkqHbdgK9b2wKA6nqRZQCgvCDm.cb2b73415c4faf214035a73b9d947c202342f3bf3bdf632132bd6d7af98cb257.ryzen","pass":"x","agent":"xmr-stak-cpu/1.3.0-1.5.0"},"id":1} + {"id":1,"jsonrpc":"2.0","error":null,"result":{"id":"479059546883218","job":{"blob":"0606e89883d205a65d8ee78991838a1cf3ec2ebbc5fb1fa43dec5fa1cd2bee4069212a549cd731000000005a88235653097aa3e97ef2ceef4aee610751a828f9be1a0758a78365fb0a4c8c05","job_id":"722134174127131","target":"dc460300"},"status":"OK"}} + {"method":"submit","params":{"id":"479059546883218","job_id":"722134174127131","nonce":"98024001","result":"c9be9381a68d533c059d614d961e0534d7d8785dd5c339c2f9596eb95f320100"},"id":1} + */ + ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_MINING, NDPI_PROTOCOL_UNKNOWN); + } + } + + ndpi_exclude_protocol(ndpi_struct, flow, NDPI_PROTOCOL_MINING, __FILE__, __FUNCTION__, __LINE__); +} + + +void init_mining_dissector(struct ndpi_detection_module_struct *ndpi_struct, + u_int32_t *id, NDPI_PROTOCOL_BITMASK *detection_bitmask) +{ + ndpi_set_bitmask_protocol_detection("Mining", ndpi_struct, detection_bitmask, *id, + NDPI_PROTOCOL_MINING, + ndpi_search_mining_tcp, + NDPI_SELECTION_BITMASK_PROTOCOL_V4_V6_TCP_WITH_PAYLOAD_WITHOUT_RETRANSMISSION, + SAVE_DETECTION_BITMASK_AS_UNKNOWN, + ADD_TO_DETECTION_BITMASK); + + *id += 1; +} + diff --git a/tests/pcap/bitcoin.pcap b/tests/pcap/bitcoin.pcap new file mode 100644 index 000000000..ce62ed779 Binary files /dev/null and b/tests/pcap/bitcoin.pcap differ diff --git a/tests/pcap/ethereum.pcap b/tests/pcap/ethereum.pcap new file mode 100644 index 000000000..f77637d62 Binary files /dev/null and b/tests/pcap/ethereum.pcap differ diff --git a/tests/pcap/monero.pcap b/tests/pcap/monero.pcap new file mode 100644 index 000000000..67d796882 Binary files /dev/null and b/tests/pcap/monero.pcap differ diff --git a/tests/pcap/zcash.pcap b/tests/pcap/zcash.pcap new file mode 100644 index 000000000..213585dbd Binary files /dev/null and b/tests/pcap/zcash.pcap differ diff --git a/tests/result/bitcoin.pcap.out b/tests/result/bitcoin.pcap.out new file mode 100644 index 000000000..d10fa583d --- /dev/null +++ b/tests/result/bitcoin.pcap.out @@ -0,0 +1,8 @@ +Mining 637 581074 6 + + 1 TCP 192.168.1.142:55328 <-> 69.118.54.122:8333 [proto: 42/Mining][cat: Unspecified/99][2 pkts/281 bytes <-> 137 pkts/191029 bytes] + 2 TCP 192.168.1.142:55348 <-> 74.89.181.229:8333 [proto: 42/Mining][cat: Unspecified/99][55 pkts/28663 bytes <-> 117 pkts/134830 bytes] + 3 TCP 192.168.1.142:55383 <-> 66.68.83.22:8333 [proto: 42/Mining][cat: Unspecified/99][65 pkts/45271 bytes <-> 96 pkts/70339 bytes] + 4 TCP 192.168.1.142:55400 <-> 195.218.16.178:8333 [proto: 42/Mining][cat: Unspecified/99][47 pkts/26824 bytes <-> 72 pkts/55927 bytes] + 5 TCP 192.168.1.142:55317 <-> 188.165.213.169:8333 [proto: 42/Mining][cat: Unspecified/99][16 pkts/21673 bytes <-> 3 pkts/1771 bytes] + 6 TCP 192.168.1.142:55487 <-> 184.58.165.119:8333 [proto: 42/Mining][cat: Unspecified/99][24 pkts/3082 bytes <-> 3 pkts/1384 bytes] diff --git a/tests/result/ethereum.pcap.out b/tests/result/ethereum.pcap.out new file mode 100644 index 000000000..a4c607d2d --- /dev/null +++ b/tests/result/ethereum.pcap.out @@ -0,0 +1,4 @@ +Mining 819 134165 2 + + 1 TCP 192.168.2.92:57726 <-> 94.23.36.128:4444 [proto: 42/Mining][cat: Unspecified/99][478 pkts/52883 bytes <-> 308 pkts/76667 bytes] + 2 TCP 192.168.2.92:41680 <-> 91.121.222.33:4444 [proto: 42/Mining][cat: Unspecified/99][20 pkts/2159 bytes <-> 13 pkts/2456 bytes] diff --git a/tests/result/monero.pcap.out b/tests/result/monero.pcap.out new file mode 100644 index 000000000..732ada2ed --- /dev/null +++ b/tests/result/monero.pcap.out @@ -0,0 +1,4 @@ +Mining 319 166676 2 + + 1 TCP 192.168.2.148:46838 <-> 94.23.199.191:3333 [proto: 42/Mining][cat: Unspecified/99][159 pkts/143155 bytes <-> 113 pkts/13204 bytes] + 2 TCP 192.168.2.148:53846 <-> 116.211.167.195:3333 [proto: 42/Mining][cat: Unspecified/99][24 pkts/4455 bytes <-> 23 pkts/5862 bytes] diff --git a/tests/result/zcash.pcap.out b/tests/result/zcash.pcap.out new file mode 100644 index 000000000..eb3e155ee --- /dev/null +++ b/tests/result/zcash.pcap.out @@ -0,0 +1,3 @@ +Mining 145 20644 1 + + 1 TCP 192.168.2.92:55190 <-> 178.32.196.217:9050 [proto: 42/Mining][cat: Unspecified/99][83 pkts/11785 bytes <-> 62 pkts/8859 bytes] -- cgit v1.2.3 From 53910f07370fac9ef7c341c16aef505fc2f691d6 Mon Sep 17 00:00:00 2001 From: Darryl Sokoloski Date: Wed, 19 Sep 2018 22:05:03 -0400 Subject: Added missing declarations for nest_log_sink dissector. Signed-off-by: Darryl Sokoloski --- src/include/ndpi_protocols.h | 2 ++ 1 file changed, 2 insertions(+) (limited to 'src/include/ndpi_protocols.h') diff --git a/src/include/ndpi_protocols.h b/src/include/ndpi_protocols.h index e1f3af4fb..48e005ce1 100644 --- a/src/include/ndpi_protocols.h +++ b/src/include/ndpi_protocols.h @@ -204,6 +204,7 @@ void ndpi_search_fix(struct ndpi_detection_module_struct *ndpi_struct, struct nd void ndpi_search_csgo(struct ndpi_detection_module_struct *ndpi_struct, struct ndpi_flow_struct *flow); void ndpi_search_ajp(struct ndpi_detection_module_struct *ndpi_struct, struct ndpi_flow_struct *flow); void ndpi_search_memcached(struct ndpi_detection_module_struct *ndpi_struct, struct ndpi_flow_struct *flow); +void ndpi_search_nest_log_sink(struct ndpi_detection_module_struct *ndpi_struct, struct ndpi_flow_struct *flow); /* --- INIT FUNCTIONS --- */ void init_diameter_dissector(struct ndpi_detection_module_struct *ndpi_struct, u_int32_t *id, NDPI_PROTOCOL_BITMASK *detection_bitmask); void init_afp_dissector(struct ndpi_detection_module_struct *ndpi_struct, u_int32_t *id, NDPI_PROTOCOL_BITMASK *detection_bitmask); @@ -359,4 +360,5 @@ void init_whatsapp_dissector(struct ndpi_detection_module_struct *ndpi_struct, u void init_ajp_dissector(struct ndpi_detection_module_struct *ndpi_struct, u_int32_t *id, NDPI_PROTOCOL_BITMASK *detection_bitmask); void init_fbzero_dissector(struct ndpi_detection_module_struct *ndpi_struct, u_int32_t *id, NDPI_PROTOCOL_BITMASK *detection_bitmask); void init_memcached_dissector(struct ndpi_detection_module_struct *ndpi_struct, u_int32_t *id, NDPI_PROTOCOL_BITMASK *detection_bitmask); +void init_nest_log_sink_dissector(struct ndpi_detection_module_struct *ndpi_struct, u_int32_t *id, NDPI_PROTOCOL_BITMASK *detection_bitmask); #endif /* __NDPI_PROTOCOLS_H__ */ -- cgit v1.2.3 From e240bbd4a4dfa34bd166ed5e9ae19a060ae42981 Mon Sep 17 00:00:00 2001 From: Luca Deri Date: Wed, 7 Nov 2018 22:43:33 +0100 Subject: Implemented UPnP dissector --- src/include/ndpi_protocols.h | 1 + src/lib/ndpi_main.c | 3 ++ src/lib/protocols/lisp.c | 22 ++++++++++++++- src/lib/protocols/upnp.c | 65 ++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 90 insertions(+), 1 deletion(-) create mode 100644 src/lib/protocols/upnp.c (limited to 'src/include/ndpi_protocols.h') diff --git a/src/include/ndpi_protocols.h b/src/include/ndpi_protocols.h index 48e005ce1..8ac7d7247 100644 --- a/src/include/ndpi_protocols.h +++ b/src/include/ndpi_protocols.h @@ -325,6 +325,7 @@ void init_tor_dissector(struct ndpi_detection_module_struct *ndpi_struct, u_int3 void init_tvants_dissector(struct ndpi_detection_module_struct *ndpi_struct, u_int32_t *id, NDPI_PROTOCOL_BITMASK *detection_bitmask); void init_tvuplayer_dissector(struct ndpi_detection_module_struct *ndpi_struct, u_int32_t *id, NDPI_PROTOCOL_BITMASK *detection_bitmask); void init_usenet_dissector(struct ndpi_detection_module_struct *ndpi_struct, u_int32_t *id, NDPI_PROTOCOL_BITMASK *detection_bitmask); +void init_upnp_dissector(struct ndpi_detection_module_struct *ndpi_struct, u_int32_t *id, NDPI_PROTOCOL_BITMASK *detection_bitmask); void init_veohtv_dissector(struct ndpi_detection_module_struct *ndpi_struct, u_int32_t *id, NDPI_PROTOCOL_BITMASK *detection_bitmask); void init_vhua_dissector(struct ndpi_detection_module_struct *ndpi_struct, u_int32_t *id, NDPI_PROTOCOL_BITMASK *detection_bitmask); void init_viber_dissector(struct ndpi_detection_module_struct *ndpi_struct, u_int32_t *id, NDPI_PROTOCOL_BITMASK *detection_bitmask); diff --git a/src/lib/ndpi_main.c b/src/lib/ndpi_main.c index 519d6a1ad..7dd8777ec 100644 --- a/src/lib/ndpi_main.c +++ b/src/lib/ndpi_main.c @@ -3170,6 +3170,9 @@ void ndpi_set_protocol_detection_bitmask2(struct ndpi_detection_module_struct *n /* REDIS */ init_redis_dissector(ndpi_struct, &a, detection_bitmask); + /* UPnP */ + init_upnp_dissector(ndpi_struct, &a, detection_bitmask); + /* VHUA */ init_vhua_dissector(ndpi_struct, &a, detection_bitmask); diff --git a/src/lib/protocols/lisp.c b/src/lib/protocols/lisp.c index 2c6d7b611..7343ca034 100644 --- a/src/lib/protocols/lisp.c +++ b/src/lib/protocols/lisp.c @@ -1,10 +1,30 @@ +/* + * list.c + * + * Copyright (C) 2017-18 - ntop.org + * + * nDPI is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * nDPI is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with nDPI. If not, see . + * + */ + #include "ndpi_protocol_ids.h" #define NDPI_CURRENT_PROTO NDPI_PROTOCOL_LISP #include "ndpi_api.h" -#define LISP_PORT 4341 +#define LISP_PORT 4341 #define LISP_PORT1 4342 static void ndpi_int_lisp_add_connection(struct ndpi_detection_module_struct *ndpi_struct, diff --git a/src/lib/protocols/upnp.c b/src/lib/protocols/upnp.c new file mode 100644 index 000000000..1706d021d --- /dev/null +++ b/src/lib/protocols/upnp.c @@ -0,0 +1,65 @@ +/* + * upnp.c + * + * Copyright (C) 2018 - ntop.org + * + * nDPI is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * nDPI is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with nDPI. If not, see . + * + */ + +#include "ndpi_protocol_ids.h" + +#define NDPI_CURRENT_PROTO NDPI_PROTOCOL_UPNP + +#include "ndpi_api.h" + +#define UPNP_PORT 3702 + +void ndpi_search_upnp(struct ndpi_detection_module_struct *ndpi_struct, + struct ndpi_flow_struct *flow) { + struct ndpi_packet_struct *packet = &flow->packet; + + NDPI_LOG_DBG(ndpi_struct, "search upnp\n"); + + if(packet->udp + && ( + (packet->iph && ((ntohl(packet->iph->daddr) & 0xF0000000) == 0xE0000000 /* A multicast address */)) +#ifdef NDPI_DETECTION_SUPPORT_IPV6 + || + (packet->iphv6 && ntohl(packet->iphv6->ip6_dst.u6_addr.u6_addr32[0]) == 0xFF020000) +#endif + ) + && (ntohs(packet->udp->dest) == UPNP_PORT) + && (packet->payload_packet_len >= 40) + && (strncmp((char*)packet->payload, " Date: Mon, 17 Dec 2018 22:33:58 +0100 Subject: Added Ookla cache --- src/include/ndpi_protocols.h | 1 + src/include/ndpi_typedefs.h | 5 +- src/lib/ndpi_main.c | 14 +- src/lib/protocols/http.c | 31 ++-- src/lib/protocols/ookla.c | 66 +++++++++ src/lib/protocols/snmp_proto.c | 1 - src/lib/third_party/include/lruc.h | 55 +++++++ src/lib/third_party/src/lruc.c | 294 +++++++++++++++++++++++++++++++++++++ 8 files changed, 452 insertions(+), 15 deletions(-) create mode 100644 src/lib/protocols/ookla.c create mode 100644 src/lib/third_party/include/lruc.h create mode 100644 src/lib/third_party/src/lruc.c (limited to 'src/include/ndpi_protocols.h') diff --git a/src/include/ndpi_protocols.h b/src/include/ndpi_protocols.h index 8ac7d7247..5349237d4 100644 --- a/src/include/ndpi_protocols.h +++ b/src/include/ndpi_protocols.h @@ -362,4 +362,5 @@ void init_ajp_dissector(struct ndpi_detection_module_struct *ndpi_struct, u_int3 void init_fbzero_dissector(struct ndpi_detection_module_struct *ndpi_struct, u_int32_t *id, NDPI_PROTOCOL_BITMASK *detection_bitmask); void init_memcached_dissector(struct ndpi_detection_module_struct *ndpi_struct, u_int32_t *id, NDPI_PROTOCOL_BITMASK *detection_bitmask); void init_nest_log_sink_dissector(struct ndpi_detection_module_struct *ndpi_struct, u_int32_t *id, NDPI_PROTOCOL_BITMASK *detection_bitmask); +void init_ookla_dissector(struct ndpi_detection_module_struct *ndpi_struct, u_int32_t *id, NDPI_PROTOCOL_BITMASK *detection_bitmask); #endif /* __NDPI_PROTOCOLS_H__ */ diff --git a/src/include/ndpi_typedefs.h b/src/include/ndpi_typedefs.h index d48453baf..294af22b3 100644 --- a/src/include/ndpi_typedefs.h +++ b/src/include/ndpi_typedefs.h @@ -996,7 +996,10 @@ struct ndpi_detection_module_struct { struct bt_announce *bt_ann; int bt_ann_len; -/* NDPI_PROTOCOL_TINC */ + /* NDPI_PROTOCOL_OOKLA */ + void *ookla_cache; + + /* NDPI_PROTOCOL_TINC */ struct cache *tinc_cache; ndpi_proto_defaults_t proto_defaults[NDPI_MAX_SUPPORTED_PROTOCOLS+NDPI_MAX_NUM_CUSTOM_PROTOCOLS]; diff --git a/src/lib/ndpi_main.c b/src/lib/ndpi_main.c index b5b3c9c6b..e882feaa6 100644 --- a/src/lib/ndpi_main.c +++ b/src/lib/ndpi_main.c @@ -30,6 +30,7 @@ #include #include "ahocorasick.h" #include "libcache.h" +#include "lruc.h" #define NDPI_CURRENT_PROTO NDPI_PROTOCOL_UNKNOWN @@ -2427,14 +2428,16 @@ void ndpi_exit_detection_module(struct ndpi_detection_module_struct *ndpi_struct ndpi_free(ndpi_struct->proto_defaults[i].protoName); } -/* NDPI_PROTOCOL_TINC */ + /* NDPI_PROTOCOL_TINC */ if(ndpi_struct->tinc_cache) cache_free((cache_t)(ndpi_struct->tinc_cache)); - if(ndpi_struct->protocols_ptree) - ndpi_Destroy_Patricia((patricia_tree_t*)ndpi_struct->protocols_ptree, - free_ptree_data); + if(ndpi_struct->ookla_cache) + lruc_free((lruc*)ndpi_struct->ookla_cache); + if(ndpi_struct->protocols_ptree) + ndpi_Destroy_Patricia((patricia_tree_t*)ndpi_struct->protocols_ptree, free_ptree_data); + if(ndpi_struct->udpRoot != NULL) ndpi_tdestroy(ndpi_struct->udpRoot, ndpi_free); if(ndpi_struct->tcpRoot != NULL) @@ -3293,6 +3296,9 @@ void ndpi_set_protocol_detection_bitmask2(struct ndpi_detection_module_struct *n /* WHATSAPP */ init_whatsapp_dissector(ndpi_struct, &a, detection_bitmask); + /* OOKLA */ + init_ookla_dissector(ndpi_struct, &a, detection_bitmask); + /* AMQP */ init_amqp_dissector(ndpi_struct, &a, detection_bitmask); diff --git a/src/lib/protocols/http.c b/src/lib/protocols/http.c index 94d918e76..09b816129 100644 --- a/src/lib/protocols/http.c +++ b/src/lib/protocols/http.c @@ -26,7 +26,7 @@ #define NDPI_CURRENT_PROTO NDPI_PROTOCOL_HTTP #include "ndpi_api.h" - +#include "lruc.h" /* global variables used for 1kxun protocol and iqiyi service */ @@ -613,7 +613,23 @@ static void ndpi_check_http_tcp(struct ndpi_detection_module_struct *ndpi_struct */ + ookla_found: ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_OOKLA, NDPI_PROTOCOL_UNKNOWN); + + if(ndpi_struct->ookla_cache == NULL) + ndpi_struct->ookla_cache = lruc_new(4*1024, 1024); + + if(ndpi_struct->ookla_cache != NULL) { + u_int8_t *dummy = (u_int8_t*)ndpi_malloc(sizeof(u_int8_t)); + + if(dummy) { + if(packet->tcp->source == htons(8080)) + lruc_set((lruc*)ndpi_struct->ookla_cache, (void*)&packet->iph->saddr, 4, dummy, 1); + else + lruc_set((lruc*)ndpi_struct->ookla_cache, (void*)&packet->iph->daddr, 4, dummy, 1); + } + } + return; } @@ -663,9 +679,8 @@ static void ndpi_check_http_tcp(struct ndpi_detection_module_struct *ndpi_struct /* Check for Ookla */ if((packet->referer_line.len > 0) - && ndpi_strnstr((const char *)packet->referer_line.ptr, "www.speedtest.net", packet->referer_line.len)) { - ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_OOKLA, NDPI_PROTOCOL_HTTP); - return; + && ndpi_strnstr((const char *)packet->referer_line.ptr, "www.speedtest.net", packet->referer_line.len)) { + goto ookla_found; } /* Check for additional field introduced by Steam */ @@ -782,17 +797,15 @@ static void ndpi_check_http_tcp(struct ndpi_detection_module_struct *ndpi_struct if((packet->payload_packet_len == 34) && (flow->l4.tcp.http_stage == 1)) { if((packet->payload[5] == ' ') && (packet->payload[9] == ' ')) { - ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_OOKLA); - return; + goto ookla_found; } } if((packet->payload_packet_len > 6) && memcmp(packet->payload, "HELLO ", 6) == 0) { /* This looks like Ookla */ - ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_OOKLA, NDPI_PROTOCOL_UNKNOWN); - return; + goto ookla_found; } else - NDPI_ADD_PROTOCOL_TO_BITMASK(flow->excluded_protocol_bitmask, NDPI_PROTOCOL_OOKLA); + NDPI_ADD_PROTOCOL_TO_BITMASK(flow->excluded_protocol_bitmask, NDPI_PROTOCOL_OOKLA); /** At first check, if this is for sure a response packet (in another direction. If not, if HTTP is detected do nothing now and return, diff --git a/src/lib/protocols/ookla.c b/src/lib/protocols/ookla.c new file mode 100644 index 000000000..b1eb295a7 --- /dev/null +++ b/src/lib/protocols/ookla.c @@ -0,0 +1,66 @@ +/* + * ookla.c + * + * Copyright (C) 2018 - ntop.org + * + * nDPI is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * nDPI is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with nDPI. If not, see . + * + */ +#include "ndpi_protocol_ids.h" + +#define NDPI_CURRENT_PROTO NDPI_PROTOCOL_OOKLA + +#include "ndpi_api.h" +#include "lruc.h" + + +void ndpi_search_ookla(struct ndpi_detection_module_struct* ndpi_struct, struct ndpi_flow_struct* flow) { + struct ndpi_packet_struct* packet = &flow->packet; + u_int32_t addr = 0; + void *value; + + NDPI_LOG_DBG(ndpi_struct, "Ookla detection\n"); + + if(packet->tcp->source == htons(8080)) + addr = packet->iph->saddr; + else if(packet->tcp->dest == htons(8080)) + addr = packet->iph->daddr; + else + goto ookla_exclude; + + if(ndpi_struct->ookla_cache != NULL) { + if(lruc_get(ndpi_struct->ookla_cache, &addr, sizeof(addr), &value) == LRUC_NO_ERROR) { + /* Don't remove it as it can be used for other connections */ + NDPI_LOG_INFO(ndpi_struct, "found ookla tcp connection\n"); + ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_OOKLA, NDPI_PROTOCOL_UNKNOWN); + return; + } + } + + ookla_exclude: + NDPI_EXCLUDE_PROTO(ndpi_struct, flow); +} + +void init_ookla_dissector(struct ndpi_detection_module_struct *ndpi_struct, + u_int32_t *id, NDPI_PROTOCOL_BITMASK *detection_bitmask) { + ndpi_set_bitmask_protocol_detection("Ookla", ndpi_struct, detection_bitmask, *id, + NDPI_PROTOCOL_OOKLA, + ndpi_search_ookla, + NDPI_SELECTION_BITMASK_PROTOCOL_TCP, + SAVE_DETECTION_BITMASK_AS_UNKNOWN, + ADD_TO_DETECTION_BITMASK); + + *id += 1; +} + diff --git a/src/lib/protocols/snmp_proto.c b/src/lib/protocols/snmp_proto.c index 759d6bae0..77ad4d233 100644 --- a/src/lib/protocols/snmp_proto.c +++ b/src/lib/protocols/snmp_proto.c @@ -123,7 +123,6 @@ void ndpi_search_snmp(struct ndpi_detection_module_struct *ndpi_struct, struct n } excl: NDPI_EXCLUDE_PROTO(ndpi_struct, flow); - } diff --git a/src/lib/third_party/include/lruc.h b/src/lib/third_party/include/lruc.h new file mode 100644 index 000000000..55fb271fe --- /dev/null +++ b/src/lib/third_party/include/lruc.h @@ -0,0 +1,55 @@ +#include +#include +#include + +#ifndef __lruc_header__ +#define __lruc_header__ + +// ------------------------------------------ +// errors +// ------------------------------------------ +typedef enum { + LRUC_NO_ERROR = 0, + LRUC_MISSING_CACHE, + LRUC_MISSING_KEY, + LRUC_MISSING_VALUE, + LRUC_PTHREAD_ERROR, + LRUC_VALUE_TOO_LARGE +} lruc_error; + + +// ------------------------------------------ +// types +// ------------------------------------------ +typedef struct { + void *value; + void *key; + uint32_t value_length; + uint32_t key_length; + uint64_t access_count; + void *next; +} lruc_item; + +typedef struct { + lruc_item **items; + uint64_t access_count; + uint64_t free_memory; + uint64_t total_memory; + uint64_t average_item_length; + uint32_t hash_table_size; + time_t seed; + lruc_item *free_items; + pthread_mutex_t *mutex; +} lruc; + + +// ------------------------------------------ +// api +// ------------------------------------------ +lruc *lruc_new(uint64_t cache_size, uint32_t average_length); +lruc_error lruc_free(lruc *cache); +lruc_error lruc_set(lruc *cache, void *key, uint32_t key_length, void *value, uint32_t value_length); +lruc_error lruc_get(lruc *cache, void *key, uint32_t key_length, void **value); +lruc_error lruc_delete(lruc *cache, void *key, uint32_t key_length); + +#endif diff --git a/src/lib/third_party/src/lruc.c b/src/lib/third_party/src/lruc.c new file mode 100644 index 000000000..f08fb2ce1 --- /dev/null +++ b/src/lib/third_party/src/lruc.c @@ -0,0 +1,294 @@ +/* https://github.com/willcannings/C-LRU-Cache */ + +#include "lruc.h" +#include +#include +#include +#include + +// ------------------------------------------ +// private functions +// ------------------------------------------ +// MurmurHash2, by Austin Appleby +// http://sites.google.com/site/murmurhash/ +uint32_t lruc_hash(lruc *cache, void *key, uint32_t key_length) { + uint32_t m = 0x5bd1e995; + uint32_t r = 24; + uint32_t h = cache->seed ^ key_length; + char* data = (char *)key; + + while(key_length >= 4) { + uint32_t k = *(uint32_t *)data; + k *= m; + k ^= k >> r; + k *= m; + h *= m; + h ^= k; + data += 4; + key_length -= 4; + } + + switch(key_length) { + case 3: h ^= data[2] << 16; + case 2: h ^= data[1] << 8; + case 1: h ^= data[0]; + h *= m; + }; + + h ^= h >> 13; + h *= m; + h ^= h >> 15; + return h % cache->hash_table_size; +} + +// compare a key against an existing item's key +int lruc_cmp_keys(lruc_item *item, void *key, uint32_t key_length) { + if(key_length != item->key_length) + return 1; + else + return memcmp(key, item->key, key_length); +} + +// remove an item and push it to the free items queue +void lruc_remove_item(lruc *cache, lruc_item *prev, lruc_item *item, uint32_t hash_index) { + if(prev) + prev->next = item->next; + else + cache->items[hash_index] = (lruc_item *) item->next; + + // free memory and update the free memory counter + cache->free_memory += item->value_length; + free(item->value); + free(item->key); + + // push the item to the free items queue + memset(item, 0, sizeof(lruc_item)); + item->next = cache->free_items; + cache->free_items = item; +} + +// remove the least recently used item +// TODO: we can optimise this by finding the n lru items, where n = required_space / average_length +void lruc_remove_lru_item(lruc *cache) { + lruc_item *min_item = NULL, *min_prev = NULL; + lruc_item *item = NULL, *prev = NULL; + uint32_t i = 0, min_index = -1; + uint64_t min_access_count = -1; + + for(; i < cache->hash_table_size; i++) { + item = cache->items[i]; + prev = NULL; + + while(item) { + if(item->access_count < min_access_count || min_access_count == -1) { + min_access_count = item->access_count; + min_item = item; + min_prev = prev; + min_index = i; + } + prev = item; + item = item->next; + } + } + + if(min_item) + lruc_remove_item(cache, min_prev, min_item, min_index); +} + +// pop an existing item off the free queue, or create a new one +lruc_item *lruc_pop_or_create_item(lruc *cache) { + lruc_item *item = NULL; + + if(cache->free_items) { + item = cache->free_items; + cache->free_items = item->next; + } else { + item = (lruc_item *) calloc(sizeof(lruc_item), 1); + } + + return item; +} + +// error helpers +#define error_for(conditions, error) if(conditions) {return error;} +#define test_for_missing_cache() error_for(!cache, LRUC_MISSING_CACHE) +#define test_for_missing_key() error_for(!key || key_length == 0, LRUC_MISSING_KEY) +#define test_for_missing_value() error_for(!value || value_length == 0, LRUC_MISSING_VALUE) +#define test_for_value_too_large() error_for(value_length > cache->total_memory, LRUC_VALUE_TOO_LARGE) + +// lock helpers +#define lock_cache() if(pthread_mutex_lock(cache->mutex)) {\ + perror("LRU Cache unable to obtain mutex lock");\ + return LRUC_PTHREAD_ERROR;\ +} + +#define unlock_cache() if(pthread_mutex_unlock(cache->mutex)) {\ + perror("LRU Cache unable to release mutex lock");\ + return LRUC_PTHREAD_ERROR;\ +} + + +// ------------------------------------------ +// public api +// ------------------------------------------ +lruc *lruc_new(uint64_t cache_size, uint32_t average_length) { + // create the cache + lruc *cache = (lruc *) calloc(sizeof(lruc), 1); + if(!cache) { + perror("LRU Cache unable to create cache object"); + return NULL; + } + cache->hash_table_size = cache_size / average_length; + cache->average_item_length = average_length; + cache->free_memory = cache_size; + cache->total_memory = cache_size; + cache->seed = time(NULL); + + // size the hash table to a guestimate of the number of slots required (assuming a perfect hash) + cache->items = (lruc_item **) calloc(sizeof(lruc_item *), cache->hash_table_size); + if(!cache->items) { + perror("LRU Cache unable to create cache hash table"); + free(cache); + return NULL; + } + + // all cache calls are guarded by a mutex + cache->mutex = (pthread_mutex_t *) malloc(sizeof(pthread_mutex_t)); + if(pthread_mutex_init(cache->mutex, NULL)) { + perror("LRU Cache unable to initialise mutex"); + free(cache->items); + free(cache); + return NULL; + } + return cache; +} + + +lruc_error lruc_free(lruc *cache) { + test_for_missing_cache(); + + // free each of the cached items, and the hash table + lruc_item *item = NULL, *next = NULL; + uint32_t i = 0; + if(cache->items) { + for(; i < cache->hash_table_size; i++) { + item = cache->items[i]; + while(item) { + next = (lruc_item *) item->next; + free(item); + item = next; + } + } + free(cache->items); + } + + // free the cache + if(cache->mutex) { + if(pthread_mutex_destroy(cache->mutex)) { + perror("LRU Cache unable to destroy mutex"); + return LRUC_PTHREAD_ERROR; + } + } + free(cache); + + return LRUC_NO_ERROR; +} + + +lruc_error lruc_set(lruc *cache, void *key, uint32_t key_length, void *value, uint32_t value_length) { + test_for_missing_cache(); + test_for_missing_key(); + test_for_missing_value(); + test_for_value_too_large(); + lock_cache(); + + // see if the key already exists + uint32_t hash_index = lruc_hash(cache, key, key_length), required = 0; + lruc_item *item = NULL, *prev = NULL; + item = cache->items[hash_index]; + + while(item && lruc_cmp_keys(item, key, key_length)) { + prev = item; + item = (lruc_item *) item->next; + } + + if(item) { + // update the value and value_lengths + required = value_length - item->value_length; + free(item->value); + item->value = value; + item->value_length = value_length; + + } else { + // insert a new item + item = lruc_pop_or_create_item(cache); + item->value = value; + item->key = key; + item->value_length = value_length; + item->key_length = key_length; + required = value_length; + + if(prev) + prev->next = item; + else + cache->items[hash_index] = item; + } + item->access_count = ++cache->access_count; + + // remove as many items as necessary to free enough space + if(required > 0 && required > cache->free_memory) { + while(cache->free_memory < required) + lruc_remove_lru_item(cache); + } + cache->free_memory -= required; + unlock_cache(); + return LRUC_NO_ERROR; +} + + +lruc_error lruc_get(lruc *cache, void *key, uint32_t key_length, void **value) { + test_for_missing_cache(); + test_for_missing_key(); + lock_cache(); + + // loop until we find the item, or hit the end of a chain + uint32_t hash_index = lruc_hash(cache, key, key_length); + lruc_item *item = cache->items[hash_index]; + + while(item && lruc_cmp_keys(item, key, key_length)) + item = (lruc_item *) item->next; + + if(item) { + *value = item->value; + item->access_count = ++cache->access_count; + } else { + *value = NULL; + } + + unlock_cache(); + return LRUC_NO_ERROR; +} + + +lruc_error lruc_delete(lruc *cache, void *key, uint32_t key_length) { + test_for_missing_cache(); + test_for_missing_key(); + lock_cache(); + + // loop until we find the item, or hit the end of a chain + lruc_item *item = NULL, *prev = NULL; + uint32_t hash_index = lruc_hash(cache, key, key_length); + item = cache->items[hash_index]; + + while(item && lruc_cmp_keys(item, key, key_length)) { + prev = item; + item = (lruc_item *) item->next; + } + + if(item) { + lruc_remove_item(cache, prev, item, hash_index); + } + + unlock_cache(); + return LRUC_NO_ERROR; +} -- cgit v1.2.3