diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/lib/ndpi_content_match.c.inc | 87 | ||||
| -rw-r--r-- | src/lib/ndpi_main.c | 86 |
2 files changed, 99 insertions, 74 deletions
diff --git a/src/lib/ndpi_content_match.c.inc b/src/lib/ndpi_content_match.c.inc index 64f1b5ee1..0d1e15ce0 100644 --- a/src/lib/ndpi_content_match.c.inc +++ b/src/lib/ndpi_content_match.c.inc @@ -7954,6 +7954,7 @@ static ndpi_network host_protocol_list[] = { { 0x0, 0, 0 } }; + /* ****************************************************** */ /* @@ -8006,44 +8007,62 @@ static ndpi_network host_protocol_list[] = { */ +/* ****************************************************** */ + +/* + +Each part of a domain name can be no longer than 63 characters. There are no single-digit top-level domains and none contain digits. It doesn't look like ICANN will approve such domains either. + +https://www.regular-expressions.info/email.html + +We also add an optional .co to match domains such as .co.uk + +(?i) is to make searches case insensitive +*/ + +#define TLD "(?i)(\\.co){0,1}\\.[a-z]{2,63}$" + +/* ****************************************************** */ + ndpi_protocol_match host_match[] = { - { "s3.ll.dash.row.aiv-cdn.net", NULL, "s3\\.ll\\.dash\\.row\\.aiv-cdn\\.net", "AmazonVideo", NDPI_PROTOCOL_AMAZON_VIDEO, NDPI_PROTOCOL_CATEGORY_STREAMING, NDPI_PROTOCOL_FUN }, + { "s3.ll.dash.row.aiv-cdn.net", NULL, "s3\\.ll\\.dash\\.row\\.aiv-cdn\\.net", "AmazonVideo", NDPI_PROTOCOL_AMAZON_VIDEO, NDPI_PROTOCOL_CATEGORY_STREAMING, NDPI_PROTOCOL_FUN }, { "s3-dub.cf.dash.row.aiv-cdn.net", NULL, "s3-dub\\.cf\\.dash\\.row\\.aiv-cdn\\.net", "AmazonVideo", NDPI_PROTOCOL_AMAZON_VIDEO, NDPI_PROTOCOL_CATEGORY_STREAMING, NDPI_PROTOCOL_FUN }, - { "dmqdd6hw24ucf.cloudfront.net", NULL, "dmqdd6hw24ucf\\.cloudfront\\.net", "AmazonVideo", NDPI_PROTOCOL_AMAZON_VIDEO, NDPI_PROTOCOL_CATEGORY_STREAMING, NDPI_PROTOCOL_FUN }, - { "d25xi40x97liuc.cloudfront.net", NULL, "d25xi40x97liuc\\.cloudfront\\.net", "AmazonVideo", NDPI_PROTOCOL_AMAZON_VIDEO, NDPI_PROTOCOL_CATEGORY_STREAMING, NDPI_PROTOCOL_FUN }, - { ".aiv-delivery.net", NULL, "\\.aiv-delivery\\.net", "AmazonVideo", NDPI_PROTOCOL_AMAZON_VIDEO, NDPI_PROTOCOL_CATEGORY_STREAMING, NDPI_PROTOCOL_FUN }, - { "amazon.", NULL, NULL, "Amazon", NDPI_PROTOCOL_AMAZON, NDPI_PROTOCOL_CATEGORY_WEB, NDPI_PROTOCOL_SAFE }, - { "amazon.com", NULL, "amazon\\.com$", "Amazon", NDPI_PROTOCOL_AMAZON, NDPI_PROTOCOL_CATEGORY_WEB, NDPI_PROTOCOL_SAFE }, - { "images-amazon.com", NULL, "images-amazon\\.com$", "Amazon", NDPI_PROTOCOL_AMAZON, NDPI_PROTOCOL_CATEGORY_WEB, NDPI_PROTOCOL_ACCEPTABLE }, - { "amazonaws.com", NULL, "amazonaws\\.com$", "Amazon", NDPI_PROTOCOL_AMAZON, NDPI_PROTOCOL_CATEGORY_WEB, NDPI_PROTOCOL_ACCEPTABLE }, - { "amazon-adsystem.com", NULL, "amazon-adsystem\\.com$", "Amazon", NDPI_PROTOCOL_AMAZON, NDPI_PROTOCOL_CATEGORY_WEB, NDPI_PROTOCOL_ACCEPTABLE }, - { ".cloudfront.net", NULL, "\\.cloudfront\\.net$", "Amazon", NDPI_PROTOCOL_AMAZON, NDPI_PROTOCOL_CATEGORY_WEB, NDPI_PROTOCOL_ACCEPTABLE }, - - { ".push.apple.com", NULL, NULL, "ApplePush", NDPI_PROTOCOL_APPLE_PUSH, NDPI_PROTOCOL_CATEGORY_CLOUD, NDPI_PROTOCOL_SAFE }, - { ".apple-dns.net", NULL, NULL, "Apple", NDPI_PROTOCOL_APPLE, NDPI_PROTOCOL_CATEGORY_WEB, NDPI_PROTOCOL_SAFE }, - { ".mzstatic.com", NULL, NULL, "Apple", NDPI_PROTOCOL_APPLE, NDPI_PROTOCOL_CATEGORY_WEB, NDPI_PROTOCOL_SAFE }, - { ".aaplimg.com", NULL, NULL, "Apple", NDPI_PROTOCOL_APPLE, NDPI_PROTOCOL_CATEGORY_WEB, NDPI_PROTOCOL_SAFE }, - { "iosapps.itunes.apple.com", NULL, NULL, "AppleStore", NDPI_PROTOCOL_APPLESTORE, NDPI_PROTOCOL_CATEGORY_SW_UPDATE, NDPI_PROTOCOL_SAFE }, /* iOS */ - { "osxapps.itunes.apple.com", NULL, NULL, "AppleStore", NDPI_PROTOCOL_APPLESTORE, NDPI_PROTOCOL_CATEGORY_SW_UPDATE, NDPI_PROTOCOL_SAFE }, /* MacOS */ - { "buy.itunes.apple.com", NULL, NULL, "AppleStore", NDPI_PROTOCOL_APPLESTORE, NDPI_PROTOCOL_CATEGORY_SW_UPDATE, NDPI_PROTOCOL_SAFE }, - { "su.itunes.apple.com", NULL, NULL, "AppleStore", NDPI_PROTOCOL_APPLESTORE, NDPI_PROTOCOL_CATEGORY_SW_UPDATE, NDPI_PROTOCOL_SAFE }, - { "se.itunes.apple.com", NULL, NULL, "AppleStore", NDPI_PROTOCOL_APPLESTORE, NDPI_PROTOCOL_CATEGORY_SW_UPDATE, NDPI_PROTOCOL_SAFE }, - { "myapp.itunes.apple.com", NULL, NULL, "AppleStore", NDPI_PROTOCOL_APPLESTORE, NDPI_PROTOCOL_CATEGORY_SW_UPDATE, NDPI_PROTOCOL_SAFE }, - { "swscan.apple.com", NULL, NULL, "AppleStore", NDPI_PROTOCOL_APPLESTORE, NDPI_PROTOCOL_CATEGORY_SW_UPDATE, NDPI_PROTOCOL_SAFE }, - { "itunes-apple.com", NULL, NULL, "AppleStore", NDPI_PROTOCOL_APPLESTORE, NDPI_PROTOCOL_CATEGORY_SW_UPDATE, NDPI_PROTOCOL_SAFE }, - { "itunes.apple.com", NULL, NULL, "AppleiTunes", NDPI_PROTOCOL_APPLE_ITUNES, NDPI_PROTOCOL_CATEGORY_STREAMING, NDPI_PROTOCOL_FUN }, - { "tlnk.io", NULL, NULL, "AppleiTunes", NDPI_PROTOCOL_APPLE_ITUNES, NDPI_PROTOCOL_CATEGORY_STREAMING, NDPI_PROTOCOL_FUN }, - { "aaplimg.com", NULL, NULL, "Apple", NDPI_PROTOCOL_APPLE, NDPI_PROTOCOL_CATEGORY_SW_UPDATE, NDPI_PROTOCOL_SAFE }, - { ".apple.com", NULL, NULL, "Apple", NDPI_PROTOCOL_APPLE, NDPI_PROTOCOL_CATEGORY_WEB, NDPI_PROTOCOL_SAFE }, - { ".icloud.com", NULL, NULL, "AppleiCloud", NDPI_PROTOCOL_APPLE_ICLOUD, NDPI_PROTOCOL_CATEGORY_WEB, NDPI_PROTOCOL_ACCEPTABLE }, - - { ".cnn.c", NULL, NULL, "CNN", NDPI_PROTOCOL_CNN, NDPI_PROTOCOL_CATEGORY_WEB, NDPI_PROTOCOL_SAFE }, + { "dmqdd6hw24ucf.cloudfront.net", NULL, "dmqdd6hw24ucf\\.cloudfront\\.net", "AmazonVideo", NDPI_PROTOCOL_AMAZON_VIDEO, NDPI_PROTOCOL_CATEGORY_STREAMING, NDPI_PROTOCOL_FUN }, + { "d25xi40x97liuc.cloudfront.net", NULL, "d25xi40x97liuc\\.cloudfront\\.net", "AmazonVideo", NDPI_PROTOCOL_AMAZON_VIDEO, NDPI_PROTOCOL_CATEGORY_STREAMING, NDPI_PROTOCOL_FUN }, + { ".aiv-delivery.net", NULL, "\\.aiv-delivery\\.net", "AmazonVideo", NDPI_PROTOCOL_AMAZON_VIDEO, NDPI_PROTOCOL_CATEGORY_STREAMING, NDPI_PROTOCOL_FUN }, + { "amazon.", NULL, NULL, "Amazon", NDPI_PROTOCOL_AMAZON, NDPI_PROTOCOL_CATEGORY_WEB, NDPI_PROTOCOL_SAFE }, + { "amazon.com", NULL, "amazon" TLD, "Amazon", NDPI_PROTOCOL_AMAZON, NDPI_PROTOCOL_CATEGORY_WEB, NDPI_PROTOCOL_SAFE }, + { "images-amazon.com", NULL, "images-amazon" TLD, "Amazon", NDPI_PROTOCOL_AMAZON, NDPI_PROTOCOL_CATEGORY_WEB, NDPI_PROTOCOL_ACCEPTABLE }, + { "amazonaws.com", NULL, "amazonaws" TLD, "Amazon", NDPI_PROTOCOL_AMAZON, NDPI_PROTOCOL_CATEGORY_WEB, NDPI_PROTOCOL_ACCEPTABLE }, + { "amazon-adsystem.com", NULL, "amazon-adsystem" TLD, "Amazon", NDPI_PROTOCOL_AMAZON, NDPI_PROTOCOL_CATEGORY_WEB, NDPI_PROTOCOL_ACCEPTABLE }, + { ".cloudfront.net", NULL, "\\.cloudfront" TLD, "Amazon", NDPI_PROTOCOL_AMAZON, NDPI_PROTOCOL_CATEGORY_WEB, NDPI_PROTOCOL_ACCEPTABLE }, + + { ".push.apple.com", NULL, "\\.push\\.apple" TLD, "ApplePush", NDPI_PROTOCOL_APPLE_PUSH, NDPI_PROTOCOL_CATEGORY_CLOUD, NDPI_PROTOCOL_SAFE }, + { ".apple-dns.net", NULL, "\\.apple-dns" TLD, "Apple", NDPI_PROTOCOL_APPLE, NDPI_PROTOCOL_CATEGORY_WEB, NDPI_PROTOCOL_SAFE }, + { ".mzstatic.com", NULL, "\\.mzstatic" TLD, "Apple", NDPI_PROTOCOL_APPLE, NDPI_PROTOCOL_CATEGORY_WEB, NDPI_PROTOCOL_SAFE }, + { ".aaplimg.com", NULL, "\\.aaplimg" TLD, "Apple", NDPI_PROTOCOL_APPLE, NDPI_PROTOCOL_CATEGORY_WEB, NDPI_PROTOCOL_SAFE }, + { "aaplimg.com", NULL, "aaplimg" TLD, "Apple", NDPI_PROTOCOL_APPLE, NDPI_PROTOCOL_CATEGORY_SW_UPDATE, NDPI_PROTOCOL_SAFE }, + { ".apple.com", NULL, "\\.apple" TLD, "Apple", NDPI_PROTOCOL_APPLE, NDPI_PROTOCOL_CATEGORY_WEB, NDPI_PROTOCOL_SAFE }, + { ".icloud.com", NULL, "\\.icloud" TLD, "AppleiCloud", NDPI_PROTOCOL_APPLE_ICLOUD, NDPI_PROTOCOL_CATEGORY_WEB, NDPI_PROTOCOL_ACCEPTABLE }, + { "iosapps.itunes.apple.com", NULL, "iosapps\\.itunes\\.apple" TLD, "AppleStore", NDPI_PROTOCOL_APPLESTORE, NDPI_PROTOCOL_CATEGORY_SW_UPDATE, NDPI_PROTOCOL_SAFE }, /* iOS */ + { "osxapps.itunes.apple.com", NULL, "osxapps\\.itunes\\.apple" TLD, "AppleStore", NDPI_PROTOCOL_APPLESTORE, NDPI_PROTOCOL_CATEGORY_SW_UPDATE, NDPI_PROTOCOL_SAFE }, /* MacOS */ + { "buy.itunes.apple.com", NULL, "buy\\.itunes\\.apple" TLD, "AppleStore", NDPI_PROTOCOL_APPLESTORE, NDPI_PROTOCOL_CATEGORY_SW_UPDATE, NDPI_PROTOCOL_SAFE }, + { "su.itunes.apple.com", NULL, "su\\.itunes\\.apple" TLD, "AppleStore", NDPI_PROTOCOL_APPLESTORE, NDPI_PROTOCOL_CATEGORY_SW_UPDATE, NDPI_PROTOCOL_SAFE }, + { "se.itunes.apple.com", NULL, "se\\.itunes\\.apple" TLD, "AppleStore", NDPI_PROTOCOL_APPLESTORE, NDPI_PROTOCOL_CATEGORY_SW_UPDATE, NDPI_PROTOCOL_SAFE }, + { "myapp.itunes.apple.com", NULL, "myapp\\.itunes\\.apple" TLD, "AppleStore", NDPI_PROTOCOL_APPLESTORE, NDPI_PROTOCOL_CATEGORY_SW_UPDATE, NDPI_PROTOCOL_SAFE }, + { "swscan.apple.com", NULL, "swscan\\.apple" TLD, "AppleStore", NDPI_PROTOCOL_APPLESTORE, NDPI_PROTOCOL_CATEGORY_SW_UPDATE, NDPI_PROTOCOL_SAFE }, + { "itunes-apple.com", NULL, "itunes-apple" TLD, "AppleStore", NDPI_PROTOCOL_APPLESTORE, NDPI_PROTOCOL_CATEGORY_SW_UPDATE, NDPI_PROTOCOL_SAFE }, + { "itunes.apple.com", NULL, "itunes\\.apple" TLD, "AppleiTunes", NDPI_PROTOCOL_APPLE_ITUNES, NDPI_PROTOCOL_CATEGORY_STREAMING, NDPI_PROTOCOL_FUN }, + { "tlnk.io", NULL, "tlnk" TLD, "AppleiTunes", NDPI_PROTOCOL_APPLE_ITUNES, NDPI_PROTOCOL_CATEGORY_STREAMING, NDPI_PROTOCOL_FUN }, + + + { ".cnn.c", NULL, "\\.cnn" TLD, "CNN", NDPI_PROTOCOL_CNN, NDPI_PROTOCOL_CATEGORY_WEB, NDPI_PROTOCOL_SAFE }, { ".cnn.net", NULL, NULL, "CNN", NDPI_PROTOCOL_CNN, NDPI_PROTOCOL_CATEGORY_WEB, NDPI_PROTOCOL_SAFE }, - { ".dropbox.com", NULL, NULL, "DropBox", NDPI_PROTOCOL_DROPBOX, NDPI_PROTOCOL_CATEGORY_CLOUD, NDPI_PROTOCOL_ACCEPTABLE }, - { ".dropboxstatic.com", NULL, NULL, "DropBox", NDPI_PROTOCOL_DROPBOX, NDPI_PROTOCOL_CATEGORY_CLOUD, NDPI_PROTOCOL_ACCEPTABLE }, - { ".dropbox-dns.com", NULL, NULL, "DropBox", NDPI_PROTOCOL_DROPBOX, NDPI_PROTOCOL_CATEGORY_CLOUD, NDPI_PROTOCOL_ACCEPTABLE }, - { "log.getdropbox.com", NULL, NULL, "DropBox", NDPI_PROTOCOL_DROPBOX, NDPI_PROTOCOL_CATEGORY_CLOUD, NDPI_PROTOCOL_ACCEPTABLE }, + { ".dropbox.com", NULL, ".\\dropbox" TLD, "DropBox", NDPI_PROTOCOL_DROPBOX, NDPI_PROTOCOL_CATEGORY_CLOUD, NDPI_PROTOCOL_ACCEPTABLE }, + { ".dropboxstatic.com", NULL, "\\.dropboxstatic" TLD, "DropBox", NDPI_PROTOCOL_DROPBOX, NDPI_PROTOCOL_CATEGORY_CLOUD, NDPI_PROTOCOL_ACCEPTABLE }, + { ".dropbox-dns.com", NULL, "\\.dropbox-dns" TLD, "DropBox", NDPI_PROTOCOL_DROPBOX, NDPI_PROTOCOL_CATEGORY_CLOUD, NDPI_PROTOCOL_ACCEPTABLE }, + { "log.getdropbox.com", NULL, "log\\.getdropbox" TLD, "DropBox", NDPI_PROTOCOL_DROPBOX, NDPI_PROTOCOL_CATEGORY_CLOUD, NDPI_PROTOCOL_ACCEPTABLE }, { ".ebay.", NULL, NULL, "eBay", NDPI_PROTOCOL_EBAY, NDPI_PROTOCOL_CATEGORY_WEB, NDPI_PROTOCOL_SAFE }, /* or FUN */ { ".ebay.com", NULL, NULL, "eBay", NDPI_PROTOCOL_EBAY, NDPI_PROTOCOL_CATEGORY_WEB, NDPI_PROTOCOL_SAFE }, diff --git a/src/lib/ndpi_main.c b/src/lib/ndpi_main.c index c61227a9f..85cd4b9eb 100644 --- a/src/lib/ndpi_main.c +++ b/src/lib/ndpi_main.c @@ -5648,9 +5648,23 @@ int ndpi_match_string_subprotocol(struct ndpi_detection_module_struct *ndpi_stru return(match.number); } -/* ****************************************************** */ +#ifdef HAVE_HYPERSCAN -#ifndef HAVE_HYPERSCAN +/* ******************************************************************** */ + +static int hyperscanEventHandler(unsigned int id, unsigned long long from, + unsigned long long to, unsigned int flags, void *ctx) { + *((int *)ctx) = (int)id; + + NDPI_LOG_DBG2(ndpi_struct, "[NDPI] Match with: %d [from: %llu][to: %llu]\n", id, from, to); + + /* return HS_SCAN_TERMINATED; */ + return 0; /* keep searching */ +} + +#endif + +/* ****************************************************** */ static int ndpi_automa_match_string_subprotocol(struct ndpi_detection_module_struct *ndpi_struct, struct ndpi_flow_struct *flow, @@ -5658,13 +5672,40 @@ static int ndpi_automa_match_string_subprotocol(struct ndpi_detection_module_str u_int16_t master_protocol_id, ndpi_protocol_match_result *ret_match, u_int8_t is_host_match) { - int matching_protocol_id; + int matching_protocol_id = NDPI_PROTOCOL_UNKNOWN; struct ndpi_packet_struct *packet = &flow->packet; +#ifndef HAVE_HYPERSCAN matching_protocol_id = ndpi_match_string_subprotocol(ndpi_struct, string_to_match, string_to_match_len, ret_match, is_host_match); - + +#else + struct hs *hs = (struct hs*)ndpi_struct->hyperscan; + hs_error_t status; + /* + TODO HYPERSCAN + In case of match fill up ret_match and set flow protocol + category + */ + status = hs_scan(hs->database, string_to_match, + string_to_match_len, 0, hs->scratch, + hyperscanEventHandler, &matching_protocol_id); + + if(status == HS_SUCCESS) { + NDPI_LOG_DBG2(ndpi_struct, "[NDPI] Hyperscan engine completed normally. Result: %s [%d][%s]\n", + ndpi_get_proto_name(ndpi_struct, matching_protocol_id), matching_protocol_id, string_to_match); + } else if(status == HS_SCAN_TERMINATED) { + NDPI_LOG_DBG2(ndpi_struct, "[NDPI] Hyperscan engine was terminated by callback. Result: %s [%d][%s]\n", + ndpi_get_proto_name(ndpi_struct, matching_protocol_id), matching_protocol_id, string_to_match); + } else { + NDPI_LOG_DBG2(ndpi_struct, "[NDPI] Hyperscan returned with error.\n"); + } + + ret_match->protocol_id = matching_protocol_id, + ret_match->protocol_category = ndpi_struct->proto_defaults[matching_protocol_id].protoCategory, + ret_match->protocol_breed = ndpi_struct->proto_defaults[matching_protocol_id].protoBreed; +#endif + #ifdef DEBUG { char m[256]; @@ -5686,7 +5727,7 @@ static int ndpi_automa_match_string_subprotocol(struct ndpi_detection_module_str flow->detected_protocol_stack[0] = packet->detected_protocol_stack[0], flow->detected_protocol_stack[1] = packet->detected_protocol_stack[1], flow->category = ret_match->protocol_category; - + return(packet->detected_protocol_stack[0]); } @@ -5698,41 +5739,6 @@ static int ndpi_automa_match_string_subprotocol(struct ndpi_detection_module_str return(NDPI_PROTOCOL_UNKNOWN); } -#else - -/* ******************************************************************** */ - -static int hyperscanEventHandler(unsigned int id, unsigned long long from, - unsigned long long to, unsigned int flags, void *ctx) { - *((int *)ctx) = (int)id; - return HS_SCAN_TERMINATED; -} - -/* *********************************************** */ - -static int ndpi_automa_match_string_subprotocol(struct ndpi_detection_module_struct *ndpi_struct, - struct ndpi_flow_struct *flow, - char *string_to_match, u_int string_to_match_len, - u_int16_t master_protocol_id, - ndpi_protocol_match_result *ret_match, - u_int8_t is_host_match) { - int rv = NDPI_PROTOCOL_UNKNOWN; - struct hs *hs = (struct hs*)ndpi_struct->hyperscan; - - /* - TODO HYPERSCAN - In case of match fill up ret_match and set flow protocol + category - */ - if(hs_scan(hs->database, string_to_match, - string_to_match_len, 0, hs->scratch, - hyperscanEventHandler, &rv) != HS_SUCCESS) - NDPI_LOG_ERR(ndpi_struct, "[NDPI] Hyperscan match returned error\n"); - - return rv; -} - -#endif - /* ****************************************************** */ int ndpi_match_host_subprotocol(struct ndpi_detection_module_struct *ndpi_struct, |