aboutsummaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
Diffstat (limited to 'src')
-rw-r--r--src/include/ndpi_api.h.in4
-rw-r--r--src/include/ndpi_protocol_ids.h2
-rw-r--r--src/include/ndpi_typedefs.h2
-rw-r--r--src/include/ndpi_win32.h3
-rw-r--r--src/lib/ndpi_community_id.c13
-rw-r--r--src/lib/ndpi_content_match.c.inc9
-rw-r--r--src/lib/ndpi_main.c58
-rw-r--r--src/lib/ndpi_utils.c6
-rw-r--r--src/lib/protocols/tls.c11
-rw-r--r--src/lib/third_party/src/hll/hll.c52
10 files changed, 103 insertions, 57 deletions
diff --git a/src/include/ndpi_api.h.in b/src/include/ndpi_api.h.in
index 0fa02e3c7..e5d2ffad3 100644
--- a/src/include/ndpi_api.h.in
+++ b/src/include/ndpi_api.h.in
@@ -903,7 +903,9 @@ extern "C" {
void ndpi_user_pwd_payload_copy(u_int8_t *dest, u_int dest_len, u_int offset,
const u_int8_t *src, u_int src_len);
u_char* ndpi_base64_decode(const u_char *src, size_t len, size_t *out_len);
- char* ndpi_base64_encode(unsigned char const* bytes_to_encode, size_t in_len);
+ char* ndpi_base64_encode(unsigned char const* bytes_to_encode, size_t in_len); /* NOTE: caller MUST free the returned pointer */
+ void ndpi_string_sha1_hash(const uint8_t *message, size_t len, u_char *hash /* 20-bytes */);
+
int ndpi_load_ipv4_ptree(struct ndpi_detection_module_struct *ndpi_str,
const char *path, u_int16_t protocol_id);
int ndpi_dpi2json(struct ndpi_detection_module_struct *ndpi_struct,
diff --git a/src/include/ndpi_protocol_ids.h b/src/include/ndpi_protocol_ids.h
index 01f54c0f9..d184ff4a5 100644
--- a/src/include/ndpi_protocol_ids.h
+++ b/src/include/ndpi_protocol_ids.h
@@ -87,7 +87,7 @@ typedef enum {
NDPI_PROTOCOL_ZATTOO = 55,
NDPI_PROTOCOL_SHOUTCAST = 56,
NDPI_PROTOCOL_SOPCAST = 57,
- NDPI_PROTOCOL_FREE_58 = 58, /* Free */
+ NDPI_PROTOCOL_DISCORD = 58,
NDPI_PROTOCOL_TVUPLAYER = 59,
NDPI_PROTOCOL_HTTP_DOWNLOAD = 60,
NDPI_PROTOCOL_QQLIVE = 61,
diff --git a/src/include/ndpi_typedefs.h b/src/include/ndpi_typedefs.h
index 66fac35af..40c27329e 100644
--- a/src/include/ndpi_typedefs.h
+++ b/src/include/ndpi_typedefs.h
@@ -79,6 +79,8 @@ typedef enum {
NDPI_SSH_OBSOLETE_CLIENT_VERSION_OR_CIPHER,
NDPI_SSH_OBSOLETE_SERVER_VERSION_OR_CIPHER,
NDPI_SMB_INSECURE_VERSION,
+ NDPI_TLS_SUSPICIOUS_ESNI_USAGE,
+ NDPI_BLACKLISTED_HOST,
/* Leave this as last member */
NDPI_MAX_RISK
diff --git a/src/include/ndpi_win32.h b/src/include/ndpi_win32.h
index 7b0b37de1..a39a2401a 100644
--- a/src/include/ndpi_win32.h
+++ b/src/include/ndpi_win32.h
@@ -76,9 +76,6 @@ typedef unsigned __int64 u_int64_t;
extern unsigned long waitForNextEvent(unsigned long ulDelay /* ms */);
#define sleep(a /* sec */) waitForNextEvent(1000*a /* ms */)
-#ifndef localtime_r
-#define localtime_r(a, b) localtime_s(b, a)
-#endif
#define strtok_r strtok_s
#define timegm _mkgmtime
diff --git a/src/lib/ndpi_community_id.c b/src/lib/ndpi_community_id.c
index 30519b59e..72f60c746 100644
--- a/src/lib/ndpi_community_id.c
+++ b/src/lib/ndpi_community_id.c
@@ -167,13 +167,15 @@ static int ndpi_community_id_peer_v4_is_less_than(u_int32_t ip1, u_int32_t ip2,
static int ndpi_community_id_peer_v6_is_less_than(struct ndpi_in6_addr *ip1, struct ndpi_in6_addr *ip2, u_int16_t p1, u_int16_t p2) {
int comp = memcmp(ip1, ip2, sizeof(struct ndpi_in6_addr));
+
return comp < 0 || (comp == 0 && p1 < p2);
}
/* **************************************************** */
-static void ndpi_community_id_sha1_hash(const uint8_t *message, size_t len, u_char *hash /* 20-bytes */) {
+void ndpi_string_sha1_hash(const uint8_t *message, size_t len, u_char *hash /* 20-bytes */) {
SHA1_CTX ctx;
+
SHA1Init(&ctx);
SHA1Update(&ctx, message, len);
SHA1Final(hash, &ctx);
@@ -185,7 +187,8 @@ static void ndpi_community_id_sha1_hash(const uint8_t *message, size_t len, u_ch
https://github.com/corelight/community-id-spec/blob/bda913f617389df07cdaa23606e11bbd318e265c/community-id.py#L285
*/
static int ndpi_community_id_finalize_and_compute_hash(u_int8_t *comm_buf, u_int16_t off, u_int8_t l4_proto,
- u_int16_t src_port, u_int16_t dst_port, char *hash_buf, u_int8_t hash_buf_len) {
+ u_int16_t src_port, u_int16_t dst_port,
+ char *hash_buf, u_int8_t hash_buf_len) {
u_int8_t pad = 0;
uint32_t hash[5];
char *community_id;
@@ -209,12 +212,12 @@ static int ndpi_community_id_finalize_and_compute_hash(u_int8_t *comm_buf, u_int
}
/* Compute SHA1 */
- ndpi_community_id_sha1_hash(comm_buf, off, (u_char*)hash);
+ ndpi_string_sha1_hash(comm_buf, off, (u_char*)hash);
/* Base64 encoding */
community_id = ndpi_base64_encode((u_int8_t*)hash, sizeof(hash));
- if (community_id == NULL)
+ if(community_id == NULL)
return -1;
#if 0 /* Debug Info */
@@ -231,7 +234,7 @@ static int ndpi_community_id_finalize_and_compute_hash(u_int8_t *comm_buf, u_int
printf("Base64: %s\n", community_id);
#endif
- if (hash_buf_len < 2 || hash_buf_len-2 < strlen(community_id)+1) {
+ if(hash_buf_len < 2 || hash_buf_len-2 < strlen(community_id)+1) {
ndpi_free(community_id);
return -1;
}
diff --git a/src/lib/ndpi_content_match.c.inc b/src/lib/ndpi_content_match.c.inc
index c8fe416eb..761ec53d5 100644
--- a/src/lib/ndpi_content_match.c.inc
+++ b/src/lib/ndpi_content_match.c.inc
@@ -9138,6 +9138,13 @@ static ndpi_protocol_match host_match[] =
{ ".net.anydesk.com", "AnyDesk", NDPI_PROTOCOL_ANYDESK, NDPI_PROTOCOL_CATEGORY_REMOTE_ACCESS, NDPI_PROTOCOL_ACCEPTABLE },
+ { "discordapp.com", "Discord", NDPI_PROTOCOL_DISCORD, NDPI_PROTOCOL_CATEGORY_COLLABORATIVE, NDPI_PROTOCOL_ACCEPTABLE },
+ { "discordapp.net", "Discord", NDPI_PROTOCOL_DISCORD, NDPI_PROTOCOL_CATEGORY_COLLABORATIVE, NDPI_PROTOCOL_ACCEPTABLE },
+ { "discord.com", "Discord", NDPI_PROTOCOL_DISCORD, NDPI_PROTOCOL_CATEGORY_COLLABORATIVE, NDPI_PROTOCOL_ACCEPTABLE },
+ { "discord.gg", "Discord", NDPI_PROTOCOL_DISCORD, NDPI_PROTOCOL_CATEGORY_COLLABORATIVE, NDPI_PROTOCOL_ACCEPTABLE },
+ { "discord.media", "Discord", NDPI_PROTOCOL_DISCORD, NDPI_PROTOCOL_CATEGORY_COLLABORATIVE, NDPI_PROTOCOL_ACCEPTABLE },
+
+
{ NULL, NULL, NDPI_PROTOCOL_UNKNOWN, NDPI_PROTOCOL_CATEGORY_UNSPECIFIED, NDPI_PROTOCOL_SAFE }
};
@@ -9341,7 +9348,7 @@ static const char *ndpi_en_popular_bigrams[] = {
static const char *ndpi_en_impossible_bigrams[] = {
"bk", "bq", "bx", "cb", "cf", "cg", "cj", "cp", "cv", "cw", "cx", "dx", "fk", "fq", "fv", "fx", /* "ee", removed it can be found in 'meeting' */
"fz", "gq", "gv", "gx", "hh", "hk", "hv", "hx", "hz", "iy", "jb", /* "jc", jcrew.com */ "jd", "jf", "jg", "jh", "jk",
- "jl", "jm", "jn", "jp", "jq", /* "jr",*/ /* "js", */ "jt", "jv", "jw", "jx", "jy", "jz", "kg", "kq", "kv", "kx",
+ "jl", "jm", "jn", "jp", "jq", /* "jr",*/ /* "js", */ "jt", "jv", "jw", "jx", "jy", "jz", /* "kg", */ "kq", "kv", "kx",
"kz", "lq", "lx", /* "mg" tamgrt.com , */ "mj", /* "mq", mqtt */ "mx", "mz", "pq", "pv", "px", "qb", "qc", "qd", "qe", "qf", "ii",
"qg", "qh", "qj", "qk", "ql", "qm", "qn", "qo", "qp", "qr", "qs", "qt", "qv", "qw", "qx", "qy", "uu",
"qz", "sx", "sz", "tq", "tx", "vb", "vc", "vd", "vf", "vg", "vh", "vj", "vm", "vn", /* "vp", Removed for vpbank.com */ "bw", /* "vk", "zr" Removed for kavkazr */
diff --git a/src/lib/ndpi_main.c b/src/lib/ndpi_main.c
index bdbdc89f3..707347c76 100644
--- a/src/lib/ndpi_main.c
+++ b/src/lib/ndpi_main.c
@@ -914,8 +914,8 @@ static void ndpi_init_protocol_defaults(struct ndpi_detection_module_struct *ndp
no_master, no_master, "Sopcast", NDPI_PROTOCOL_CATEGORY_VIDEO,
ndpi_build_default_ports(ports_a, 0, 0, 0, 0, 0) /* TCP */,
ndpi_build_default_ports(ports_b, 0, 0, 0, 0, 0) /* UDP */);
- ndpi_set_proto_defaults(ndpi_str, NDPI_PROTOCOL_FUN, NDPI_PROTOCOL_FREE_58, 0 /* can_have_a_subprotocol */,
- no_master, no_master, "Free58", NDPI_PROTOCOL_CATEGORY_VIDEO,
+ ndpi_set_proto_defaults(ndpi_str, NDPI_PROTOCOL_FUN, NDPI_PROTOCOL_DISCORD, 0 /* can_have_a_subprotocol */,
+ no_master, no_master, "Discord", NDPI_PROTOCOL_CATEGORY_COLLABORATIVE,
ndpi_build_default_ports(ports_a, 0, 0, 0, 0, 0) /* TCP */,
ndpi_build_default_ports(ports_b, 0, 0, 0, 0, 0) /* UDP */);
ndpi_set_proto_defaults(ndpi_str, NDPI_PROTOCOL_FUN, NDPI_PROTOCOL_TVUPLAYER, 0 /* can_have_a_subprotocol */,
@@ -2591,11 +2591,17 @@ int ndpi_handle_rule(struct ndpi_detection_module_struct *ndpi_str, char *rule,
is_ip = 1, value = &attr[3];
else if(strncmp(attr, "host:", 5) == 0) {
/* host:"<value>",host:"<value>",.....@<subproto> */
+ u_int i, max_len;
+
value = &attr[5];
if(value[0] == '"')
value++; /* remove leading " */
- if(value[strlen(value) - 1] == '"')
- value[strlen(value) - 1] = '\0'; /* remove trailing " */
+
+ max_len = strlen(value) - 1;
+ if(value[max_len] == '"')
+ value[max_len] = '\0'; /* remove trailing " */
+
+ for(i=0; i<max_len; i++) value[i] = tolower(value[i]);
}
if(is_tcp || is_udp) {
@@ -4730,21 +4736,34 @@ ndpi_protocol ndpi_detection_process_packet(struct ndpi_detection_module_struct
if(found
&& (found->proto->protoId != NDPI_PROTOCOL_UNKNOWN)
- && (found->proto->protoId != ret.master_protocol)) {
+ && (found->proto->protoId != ret.master_protocol)
+ && (found->proto->protoId != ret.app_protocol)
+ ) {
// printf("******** %u / %u\n", found->proto->protoId, ret.master_protocol);
if(!ndpi_check_protocol_port_mismatch_exceptions(ndpi_str, flow, found, &ret))
NDPI_SET_BIT(flow->risk, NDPI_KNOWN_PROTOCOL_ON_NON_STANDARD_PORT);
} else if(default_ports && (default_ports[0] != 0)) {
- u_int8_t found = 0, i;
+ u_int8_t found = 0, i, num_loops = 0;
+ check_default_ports:
for(i=0; (i<MAX_DEFAULT_PORTS) && (default_ports[i] != 0); i++) {
if((default_ports[i] == sport) || (default_ports[i] == dport)) {
found = 1;
break;
- }
+ }
} /* for */
+ if((num_loops == 0) && (!found)) {
+ if(flow->packet.udp)
+ default_ports = ndpi_str->proto_defaults[ret.app_protocol].udp_default_ports;
+ else
+ default_ports = ndpi_str->proto_defaults[ret.app_protocol].tcp_default_ports;
+
+ num_loops = 1;
+ goto check_default_ports;
+ }
+
if(!found) {
// printf("******** Invalid default port\n");
NDPI_SET_BIT(flow->risk, NDPI_KNOWN_PROTOCOL_ON_NON_STANDARD_PORT);
@@ -6105,7 +6124,7 @@ u_int16_t ndpi_match_host_subprotocol(struct ndpi_detection_module_struct *ndpi_
int ndpi_match_hostname_protocol(struct ndpi_detection_module_struct *ndpi_struct, struct ndpi_flow_struct *flow,
u_int16_t master_protocol, char *name, u_int name_len) {
ndpi_protocol_match_result ret_match;
- u_int16_t subproto, what_len;
+ u_int16_t subproto, what_len, i;
char *what;
if((name_len > 2) && (name[0] == '*') && (name[1] == '.'))
@@ -6113,6 +6132,9 @@ int ndpi_match_hostname_protocol(struct ndpi_detection_module_struct *ndpi_struc
else
what = name, what_len = name_len;
+ /* Convert it first to lowercase: we assume meory is writable as in nDPI dissctors */
+ for(i=0; i<name_len; i++) what[i] = tolower(what[i]);
+
subproto = ndpi_match_host_subprotocol(ndpi_struct, flow, what, what_len, &ret_match, master_protocol);
if(subproto != NDPI_PROTOCOL_UNKNOWN) {
@@ -6570,7 +6592,7 @@ static int enough(int a, int b) {
/* ******************************************************************** */
-// #define DGA_DEBUG 1
+/* #define DGA_DEBUG 1 */
int ndpi_check_dga_name(struct ndpi_detection_module_struct *ndpi_str,
struct ndpi_flow_struct *flow,
@@ -6647,18 +6669,16 @@ int ndpi_check_dga_name(struct ndpi_detection_module_struct *ndpi_str,
printf("-> Checking %c%c\n", word[i], word[i+1]);
#endif
- if(ndpi_match_bigram(ndpi_str, &ndpi_str->bigrams_automa, &word[i])) {
- num_found++;
- } else {
- if(ndpi_match_bigram(ndpi_str,
- &ndpi_str->impossible_bigrams_automa,
- &word[i])) {
+ if(ndpi_match_bigram(ndpi_str,
+ &ndpi_str->impossible_bigrams_automa,
+ &word[i])) {
#ifdef DGA_DEBUG
- printf("IMPOSSIBLE %s\n", &word[i]);
+ printf("IMPOSSIBLE %s\n", &word[i]);
#endif
- num_impossible++;
- }
- }
+ num_impossible++;
+ } else if(ndpi_match_bigram(ndpi_str, &ndpi_str->bigrams_automa, &word[i])) {
+ num_found++;
+ }
} /* for */
} /* for */
diff --git a/src/lib/ndpi_utils.c b/src/lib/ndpi_utils.c
index 347e65d52..0d2f5cf3c 100644
--- a/src/lib/ndpi_utils.c
+++ b/src/lib/ndpi_utils.c
@@ -874,6 +874,7 @@ u_char* ndpi_base64_decode(const u_char *src, size_t len, size_t *out_len) {
/* ********************************** */
+/* NOTE: caller MUST free returned pointer */
char* ndpi_base64_encode(unsigned char const* bytes_to_encode, size_t in_len) {
size_t len = 0, ret_size;
char *ret;
@@ -1532,7 +1533,10 @@ const char* ndpi_risk2str(ndpi_risk_enum risk) {
case NDPI_SMB_INSECURE_VERSION:
return("SMB Insecure Version");
-
+
+ case NDPI_TLS_SUSPICIOUS_ESNI_USAGE:
+ return("TLS Suspicious ESNI Usage");
+
default:
snprintf(buf, sizeof(buf), "%d", (int)risk);
return(buf);
diff --git a/src/lib/protocols/tls.c b/src/lib/protocols/tls.c
index ec267ba5e..883de7666 100644
--- a/src/lib/protocols/tls.c
+++ b/src/lib/protocols/tls.c
@@ -316,7 +316,9 @@ static void processCertificateElements(struct ndpi_detection_module_struct *ndpi
printf("[TLS] %s() IssuerDN [%s]\n", __FUNCTION__, rdnSeqBuf);
#endif
- if(rdn_len) flow->protos.stun_ssl.ssl.issuerDN = ndpi_strdup(rdnSeqBuf);
+ if(rdn_len && (flow->protos.stun_ssl.ssl.issuerDN == NULL))
+ flow->protos.stun_ssl.ssl.issuerDN = ndpi_strdup(rdnSeqBuf);
+
rdn_len = 0; /* Reset buffer */
}
@@ -1432,6 +1434,13 @@ int processClientServerHello(struct ndpi_detection_module_struct *ndpi_struct,
NDPI_SET_BIT(flow->risk, NDPI_TLS_NOT_CARRYING_HTTPS);
}
+ /* Suspicious Domain Fronting:
+ https://github.com/SixGenInc/Noctilucent/blob/master/docs/ */
+ if(flow->protos.stun_ssl.ssl.encrypted_sni.esni &&
+ flow->protos.stun_ssl.ssl.client_requested_server_name[0] != '\0') {
+ NDPI_SET_BIT(flow->risk, NDPI_TLS_SUSPICIOUS_ESNI_USAGE);
+ }
+
return(2 /* Client Certificate */);
} else {
#ifdef DEBUG_TLS
diff --git a/src/lib/third_party/src/hll/hll.c b/src/lib/third_party/src/hll/hll.c
index a7006c7ed..c526c6af0 100644
--- a/src/lib/third_party/src/hll/hll.c
+++ b/src/lib/third_party/src/hll/hll.c
@@ -34,6 +34,7 @@ u_int32_t _hll_hash(const struct ndpi_hll *hll) {
return MurmurHash3_x86_32(hll->registers, (u_int32_t)hll->size, 0);
}
+/* Count the number of leading zero's */
static __inline u_int8_t _hll_rank(u_int32_t hash, u_int8_t bits) {
u_int8_t i;
@@ -48,24 +49,26 @@ static __inline u_int8_t _hll_rank(u_int32_t hash, u_int8_t bits) {
}
/*
- IMPORTANT: memory usage notes
+ IMPORTANT: HyperLogLog Memory and StandardError Notes
- [i: 4] 16 bytes
- [i: 5] 32 bytes
- [i: 6] 64 bytes
- [i: 7] 128 bytes
- [i: 8] 256 bytes
- [i: 9] 512 bytes
- [i: 10] 1024 bytes
- [i: 11] 2048 bytes
- [i: 12] 4096 bytes
- [i: 13] 8192 bytes
- [i: 14] 16384 bytes
- [i: 15] 32768 bytes
- [i: 16] 65536 bytes
- [i: 17] 131072 bytes
- [i: 18] 262144 bytes
- [i: 19] 524288 bytes
+ StdError = 1.04/sqrt(2^i)
+
+ [i: 4] 16 bytes [StdError: 26% ]
+ [i: 5] 32 bytes [StdError: 18.4%]
+ [i: 6] 64 bytes [StdError: 13% ]
+ [i: 7] 128 bytes [StdError: 9.2% ]
+ [i: 8] 256 bytes [StdError: 6.5% ]
+ [i: 9] 512 bytes [StdError: 4.6% ]
+ [i: 10] 1024 bytes [StdError: 3.25%]
+ [i: 11] 2048 bytes [StdError: 2.3% ]
+ [i: 12] 4096 bytes [StdError: 1.6% ]
+ [i: 13] 8192 bytes [StdError: 1.15%]
+ [i: 14] 16384 bytes [StdError: 0.81%]
+ [i: 15] 32768 bytes [StdError: 0.57%]
+ [i: 16] 65536 bytes [StdError: 0.41%]
+ [i: 17] 131072 bytes [StdError: 0.29%]
+ [i: 18] 262144 bytes [StdError: 0.2% ]
+ [i: 19] 524288 bytes [StdError: 0.14%]
*/
int hll_init(struct ndpi_hll *hll, u_int8_t bits) {
if(bits < 4 || bits > 20) {
@@ -73,9 +76,9 @@ int hll_init(struct ndpi_hll *hll, u_int8_t bits) {
return -1;
}
- hll->bits = bits;
- hll->size = (size_t)1 << bits;
- hll->registers = ndpi_calloc(hll->size, 1);
+ hll->bits = bits; /* Number of bits of buckets number */
+ hll->size = (size_t)1 << bits; /* Number of buckets 2^bits */
+ hll->registers = ndpi_calloc(hll->size, 1); /* Create the bucket register counters */
/* printf("%lu bytes\n", hll->size); */
return 0;
@@ -96,12 +99,11 @@ void hll_reset(struct ndpi_hll *hll) {
static __inline void _hll_add_hash(struct ndpi_hll *hll, u_int32_t hash) {
if(hll->registers) {
- u_int32_t index = hash >> (32 - hll->bits);
- u_int8_t rank = _hll_rank(hash, hll->bits);
+ u_int32_t index = hash >> (32 - hll->bits); /* Use the first 'hll->bits' bits as bucket index */
+ u_int8_t rank = _hll_rank(hash, hll->bits); /* Count the number of leading 0 */
- if(rank > hll->registers[index]) {
- hll->registers[index] = rank;
- }
+ if(rank > hll->registers[index])
+ hll->registers[index] = rank; /* Store the largest number of lesding zeros for the bucket */
}
}
Powered by cgit, Themed by Ted Yin.