From f3b0878a37d252f0a9300eab7a7df28d64146211 Mon Sep 17 00:00:00 2001 From: Luca Deri Date: Thu, 17 Jan 2019 00:39:45 +0100 Subject: Warnign fixes Updated test results --- src/lib/protocols/ssl.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/lib/protocols/ssl.c') diff --git a/src/lib/protocols/ssl.c b/src/lib/protocols/ssl.c index 25d535a57..4651b358f 100644 --- a/src/lib/protocols/ssl.c +++ b/src/lib/protocols/ssl.c @@ -559,7 +559,7 @@ static u_int8_t ndpi_search_sslv3_direction1(struct ndpi_detection_module_struct } } - if((packet->payload_packet_len > temp && packet->payload_packet_len > 100) && packet->payload_packet_len > 9) { + if((packet->payload_packet_len > temp) && (packet->payload_packet_len > 100)) { /* the server hello may be split into small packets and the certificate has its own SSL Record * so temp contains only the length for the first ServerHello block */ u_int32_t cert_start; -- cgit v1.2.3 From efef99cbadc8ddd6f6743e04d184fe240d6eb334 Mon Sep 17 00:00:00 2001 From: Luca Deri Date: Thu, 17 Jan 2019 00:40:28 +0100 Subject: Removed this party LRU and replaced with home grown --- example/ndpiReader.c | 5 +- src/include/Makefile.am | 2 +- src/include/ndpi_api.h | 6 + src/include/ndpi_typedefs.h | 6 +- src/lib/ndpi_main.c | 48 +++++- src/lib/protocols/http.c | 19 +-- src/lib/protocols/ookla.c | 17 +-- src/lib/protocols/ssl.c | 4 + src/lib/third_party/include/lruc.h | 55 ------- src/lib/third_party/src/lruc.c | 294 ------------------------------------- 10 files changed, 79 insertions(+), 377 deletions(-) delete mode 100644 src/lib/third_party/include/lruc.h delete mode 100644 src/lib/third_party/src/lruc.c (limited to 'src/lib/protocols/ssl.c') diff --git a/example/ndpiReader.c b/example/ndpiReader.c index 6c3dfeee8..8996ea1a5 100644 --- a/example/ndpiReader.c +++ b/example/ndpiReader.c @@ -761,6 +761,9 @@ static void printFlow(u_int16_t id, struct ndpi_flow_info *flow, u_int16_t threa if((verbose != 1) && (verbose != 2)) return; + if(5222 == ntohs(flow->dst_port)) + printf("************\n"); + if(!json_flag) { fprintf(out, "\t%u", id); @@ -913,7 +916,7 @@ static void node_proto_guess_walker(const void *node, ndpi_VISIT which, int dept struct ndpi_flow_info *flow = *(struct ndpi_flow_info **) node; u_int16_t thread_id = *((u_int16_t *) user_data); - if((which == ndpi_preorder) || (which == ndpi_leaf)) { /* Avoid walking the same node multiple times */ + if((which == ndpi_preorder) || (which == ndpi_leaf)) { /* Avoid walking the same node multiple times */ if((!flow->detection_completed) && flow->ndpi_flow) flow->detected_protocol = ndpi_detection_giveup(ndpi_thread_info[0].workflow->ndpi_struct, flow->ndpi_flow, enable_protocol_guess); diff --git a/src/include/Makefile.am b/src/include/Makefile.am index 6eeca93b4..47fcbd224 100644 --- a/src/include/Makefile.am +++ b/src/include/Makefile.am @@ -1,4 +1,4 @@ -library_includedir=$(includedir)/libndpi-@VERSION@/libndpi +plibrary_includedir=$(includedir)/libndpi-@VERSION@/libndpi library_include_HEADERS = ndpi_api.h \ ndpi_define.h \ diff --git a/src/include/ndpi_api.h b/src/include/ndpi_api.h index e57f3a568..e09c91c8e 100644 --- a/src/include/ndpi_api.h +++ b/src/include/ndpi_api.h @@ -756,6 +756,12 @@ extern "C" { u_int ndpi_get_ndpi_num_custom_protocols(struct ndpi_detection_module_struct *ndpi_mod); u_int ndpi_get_ndpi_detection_module_size(); void ndpi_set_log_level(struct ndpi_detection_module_struct *ndpi_mod, u_int l); + + /* LRU cache */ + struct ndpi_lru_cache* ndpi_lru_cache_init(u_int32_t num_entries); + void ndpi_lru_free_cache(struct ndpi_lru_cache *c); + u_int8_t ndpi_lru_find_cache(struct ndpi_lru_cache *c, u_int32_t key, u_int8_t clean_key_when_found); + void ndpi_lru_add_to_cache(struct ndpi_lru_cache *c, u_int32_t key); /** * Add a string to match to an automata diff --git a/src/include/ndpi_typedefs.h b/src/include/ndpi_typedefs.h index 34d308e2e..cf1af8bc3 100644 --- a/src/include/ndpi_typedefs.h +++ b/src/include/ndpi_typedefs.h @@ -367,6 +367,10 @@ typedef enum { HTTP_METHOD_CONNECT } ndpi_http_method; +struct ndpi_lru_cache { + u_int32_t num_entries, *entries; +}; + struct ndpi_id_struct { /** detected_protocol_bitmask: @@ -997,7 +1001,7 @@ struct ndpi_detection_module_struct { int bt_ann_len; /* NDPI_PROTOCOL_OOKLA */ - void *ookla_cache; + struct ndpi_lru_cache *ookla_cache; /* NDPI_PROTOCOL_TINC */ struct cache *tinc_cache; diff --git a/src/lib/ndpi_main.c b/src/lib/ndpi_main.c index 863943c4a..594d521d3 100644 --- a/src/lib/ndpi_main.c +++ b/src/lib/ndpi_main.c @@ -30,7 +30,6 @@ #include #include "ahocorasick.h" #include "libcache.h" -#include "lruc.h" #define NDPI_CURRENT_PROTO NDPI_PROTOCOL_UNKNOWN @@ -2433,7 +2432,7 @@ void ndpi_exit_detection_module(struct ndpi_detection_module_struct *ndpi_struct cache_free((cache_t)(ndpi_struct->tinc_cache)); if(ndpi_struct->ookla_cache) - lruc_free((lruc*)ndpi_struct->ookla_cache); + ndpi_lru_free_cache(ndpi_struct->ookla_cache); if(ndpi_struct->protocols_ptree) ndpi_Destroy_Patricia((patricia_tree_t*)ndpi_struct->protocols_ptree, free_ptree_data); @@ -3995,6 +3994,9 @@ ndpi_protocol ndpi_detection_giveup(struct ndpi_detection_module_struct *ndpi_st if(flow == NULL) return(ret); + if(flow->packet.tcp && (5222 == ntohs(flow->packet.tcp->dest))) + printf("%u - %u\n", ntohs(flow->packet.tcp->source), ntohs(flow->packet.tcp->dest)); + /* TODO: add the remaining stage_XXXX protocols */ if(flow->detected_protocol_stack[0] == NDPI_PROTOCOL_UNKNOWN) { u_int16_t guessed_protocol_id, guessed_host_protocol_id; @@ -6171,6 +6173,48 @@ void ndpi_set_log_level(struct ndpi_detection_module_struct *ndpi_mod, u_int l) /* ******************************************************************** */ +/* LRU cache */ + +struct ndpi_lru_cache* ndpi_lru_cache_init(u_int32_t num_entries) { + struct ndpi_lru_cache *c = (struct ndpi_lru_cache*)malloc(sizeof(struct ndpi_lru_cache)); + + if(!c) return(NULL); + + c->entries = (u_int32_t*)calloc(num_entries, sizeof(u_int32_t)); + + if(!c->entries) { + free(c); + return(NULL); + } else + c->num_entries = num_entries; + + return(c); +} + +void ndpi_lru_free_cache(struct ndpi_lru_cache *c) { + free(c->entries); + free(c); +} + + +u_int8_t ndpi_lru_find_cache(struct ndpi_lru_cache *c, u_int32_t key, u_int8_t clean_key_when_found) { + u_int32_t slot = key % c->num_entries; + + if(c->entries[slot] == key) { + if(clean_key_when_found) c->entries[slot] = 0; + return(1); + } else + return(0); +} + +void ndpi_lru_add_to_cache(struct ndpi_lru_cache *c, u_int32_t key) { + u_int32_t slot = key % c->num_entries; + + c->entries[slot] = key; +} + +/* ******************************************************************** */ + /* NOTE: - Leave fields empty/zero when information is missing (e.g. with ICMP ports are zero) diff --git a/src/lib/protocols/http.c b/src/lib/protocols/http.c index 661e55732..16b122d06 100644 --- a/src/lib/protocols/http.c +++ b/src/lib/protocols/http.c @@ -26,7 +26,6 @@ #define NDPI_CURRENT_PROTO NDPI_PROTOCOL_HTTP #include "ndpi_api.h" -#include "lruc.h" static void ndpi_int_http_add_connection(struct ndpi_detection_module_struct *ndpi_struct, struct ndpi_flow_struct *flow, @@ -642,19 +641,15 @@ static void ndpi_check_http_tcp(struct ndpi_detection_module_struct *ndpi_struct ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_OOKLA, NDPI_PROTOCOL_UNKNOWN); if(ndpi_struct->ookla_cache == NULL) - ndpi_struct->ookla_cache = lruc_new(4*1024, 1024); - + ndpi_struct->ookla_cache = ndpi_lru_cache_init(1024); + if(packet->iph != NULL && ndpi_struct->ookla_cache != NULL) { - u_int8_t *dummy = (u_int8_t*)ndpi_malloc(sizeof(u_int8_t)); - - if(dummy) { - if(packet->tcp->source == htons(8080)) - lruc_set((lruc*)ndpi_struct->ookla_cache, (void*)&packet->iph->saddr, 4, dummy, 1); - else - lruc_set((lruc*)ndpi_struct->ookla_cache, (void*)&packet->iph->daddr, 4, dummy, 1); - } + if(packet->tcp->source == htons(8080)) + ndpi_lru_add_to_cache(ndpi_struct->ookla_cache, packet->iph->saddr); + else + ndpi_lru_add_to_cache(ndpi_struct->ookla_cache, packet->iph->daddr); } - + return; } diff --git a/src/lib/protocols/ookla.c b/src/lib/protocols/ookla.c index 4d46dbf58..06d97e216 100644 --- a/src/lib/protocols/ookla.c +++ b/src/lib/protocols/ookla.c @@ -22,36 +22,32 @@ #define NDPI_CURRENT_PROTO NDPI_PROTOCOL_OOKLA #include "ndpi_api.h" -#include "lruc.h" void ndpi_search_ookla(struct ndpi_detection_module_struct* ndpi_struct, struct ndpi_flow_struct* flow) { -#if 0 struct ndpi_packet_struct* packet = &flow->packet; u_int32_t addr = 0; void *value; - + NDPI_LOG_DBG(ndpi_struct, "Ookla detection\n"); - + if(packet->tcp->source == htons(8080)) addr = packet->iph->saddr; else if(packet->tcp->dest == htons(8080)) addr = packet->iph->daddr; else goto ookla_exclude; - + if(ndpi_struct->ookla_cache != NULL) { - if(lruc_get(ndpi_struct->ookla_cache, &addr, sizeof(addr), &value) == LRUC_NO_ERROR) { - /* Don't remove it as it can be used for other connections */ + if(ndpi_lru_find_cache(ndpi_struct->ookla_cache, addr, 0 /* Don't remove it as it can be used for other connections */)) { NDPI_LOG_INFO(ndpi_struct, "found ookla tcp connection\n"); ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_OOKLA, NDPI_PROTOCOL_UNKNOWN); return; - } + } } ookla_exclude: -#endif - NDPI_EXCLUDE_PROTO(ndpi_struct, flow); + NDPI_EXCLUDE_PROTO(ndpi_struct, flow); } void init_ookla_dissector(struct ndpi_detection_module_struct *ndpi_struct, @@ -65,4 +61,3 @@ void init_ookla_dissector(struct ndpi_detection_module_struct *ndpi_struct, *id += 1; } - diff --git a/src/lib/protocols/ssl.c b/src/lib/protocols/ssl.c index 25d535a57..785ffa70d 100644 --- a/src/lib/protocols/ssl.c +++ b/src/lib/protocols/ssl.c @@ -626,6 +626,10 @@ void ndpi_search_ssl_tcp(struct ndpi_detection_module_struct *ndpi_struct, struc struct ndpi_packet_struct *packet = &flow->packet; u_int8_t ret; + if(flow->packet.tcp && (5222 == ntohs(flow->packet.tcp->dest))) + printf("%u - %u\n", ntohs(flow->packet.tcp->source), ntohs(flow->packet.tcp->dest)); + + if(packet->detected_protocol_stack[0] == NDPI_PROTOCOL_SSL) { if(flow->l4.tcp.ssl_stage == 3 && packet->payload_packet_len > 20 && flow->packet_counter < 5) { /* this should only happen, when we detected SSL with a packet that had parts of the certificate in subsequent packets diff --git a/src/lib/third_party/include/lruc.h b/src/lib/third_party/include/lruc.h deleted file mode 100644 index 55fb271fe..000000000 --- a/src/lib/third_party/include/lruc.h +++ /dev/null @@ -1,55 +0,0 @@ -#include -#include -#include - -#ifndef __lruc_header__ -#define __lruc_header__ - -// ------------------------------------------ -// errors -// ------------------------------------------ -typedef enum { - LRUC_NO_ERROR = 0, - LRUC_MISSING_CACHE, - LRUC_MISSING_KEY, - LRUC_MISSING_VALUE, - LRUC_PTHREAD_ERROR, - LRUC_VALUE_TOO_LARGE -} lruc_error; - - -// ------------------------------------------ -// types -// ------------------------------------------ -typedef struct { - void *value; - void *key; - uint32_t value_length; - uint32_t key_length; - uint64_t access_count; - void *next; -} lruc_item; - -typedef struct { - lruc_item **items; - uint64_t access_count; - uint64_t free_memory; - uint64_t total_memory; - uint64_t average_item_length; - uint32_t hash_table_size; - time_t seed; - lruc_item *free_items; - pthread_mutex_t *mutex; -} lruc; - - -// ------------------------------------------ -// api -// ------------------------------------------ -lruc *lruc_new(uint64_t cache_size, uint32_t average_length); -lruc_error lruc_free(lruc *cache); -lruc_error lruc_set(lruc *cache, void *key, uint32_t key_length, void *value, uint32_t value_length); -lruc_error lruc_get(lruc *cache, void *key, uint32_t key_length, void **value); -lruc_error lruc_delete(lruc *cache, void *key, uint32_t key_length); - -#endif diff --git a/src/lib/third_party/src/lruc.c b/src/lib/third_party/src/lruc.c deleted file mode 100644 index f08fb2ce1..000000000 --- a/src/lib/third_party/src/lruc.c +++ /dev/null @@ -1,294 +0,0 @@ -/* https://github.com/willcannings/C-LRU-Cache */ - -#include "lruc.h" -#include -#include -#include -#include - -// ------------------------------------------ -// private functions -// ------------------------------------------ -// MurmurHash2, by Austin Appleby -// http://sites.google.com/site/murmurhash/ -uint32_t lruc_hash(lruc *cache, void *key, uint32_t key_length) { - uint32_t m = 0x5bd1e995; - uint32_t r = 24; - uint32_t h = cache->seed ^ key_length; - char* data = (char *)key; - - while(key_length >= 4) { - uint32_t k = *(uint32_t *)data; - k *= m; - k ^= k >> r; - k *= m; - h *= m; - h ^= k; - data += 4; - key_length -= 4; - } - - switch(key_length) { - case 3: h ^= data[2] << 16; - case 2: h ^= data[1] << 8; - case 1: h ^= data[0]; - h *= m; - }; - - h ^= h >> 13; - h *= m; - h ^= h >> 15; - return h % cache->hash_table_size; -} - -// compare a key against an existing item's key -int lruc_cmp_keys(lruc_item *item, void *key, uint32_t key_length) { - if(key_length != item->key_length) - return 1; - else - return memcmp(key, item->key, key_length); -} - -// remove an item and push it to the free items queue -void lruc_remove_item(lruc *cache, lruc_item *prev, lruc_item *item, uint32_t hash_index) { - if(prev) - prev->next = item->next; - else - cache->items[hash_index] = (lruc_item *) item->next; - - // free memory and update the free memory counter - cache->free_memory += item->value_length; - free(item->value); - free(item->key); - - // push the item to the free items queue - memset(item, 0, sizeof(lruc_item)); - item->next = cache->free_items; - cache->free_items = item; -} - -// remove the least recently used item -// TODO: we can optimise this by finding the n lru items, where n = required_space / average_length -void lruc_remove_lru_item(lruc *cache) { - lruc_item *min_item = NULL, *min_prev = NULL; - lruc_item *item = NULL, *prev = NULL; - uint32_t i = 0, min_index = -1; - uint64_t min_access_count = -1; - - for(; i < cache->hash_table_size; i++) { - item = cache->items[i]; - prev = NULL; - - while(item) { - if(item->access_count < min_access_count || min_access_count == -1) { - min_access_count = item->access_count; - min_item = item; - min_prev = prev; - min_index = i; - } - prev = item; - item = item->next; - } - } - - if(min_item) - lruc_remove_item(cache, min_prev, min_item, min_index); -} - -// pop an existing item off the free queue, or create a new one -lruc_item *lruc_pop_or_create_item(lruc *cache) { - lruc_item *item = NULL; - - if(cache->free_items) { - item = cache->free_items; - cache->free_items = item->next; - } else { - item = (lruc_item *) calloc(sizeof(lruc_item), 1); - } - - return item; -} - -// error helpers -#define error_for(conditions, error) if(conditions) {return error;} -#define test_for_missing_cache() error_for(!cache, LRUC_MISSING_CACHE) -#define test_for_missing_key() error_for(!key || key_length == 0, LRUC_MISSING_KEY) -#define test_for_missing_value() error_for(!value || value_length == 0, LRUC_MISSING_VALUE) -#define test_for_value_too_large() error_for(value_length > cache->total_memory, LRUC_VALUE_TOO_LARGE) - -// lock helpers -#define lock_cache() if(pthread_mutex_lock(cache->mutex)) {\ - perror("LRU Cache unable to obtain mutex lock");\ - return LRUC_PTHREAD_ERROR;\ -} - -#define unlock_cache() if(pthread_mutex_unlock(cache->mutex)) {\ - perror("LRU Cache unable to release mutex lock");\ - return LRUC_PTHREAD_ERROR;\ -} - - -// ------------------------------------------ -// public api -// ------------------------------------------ -lruc *lruc_new(uint64_t cache_size, uint32_t average_length) { - // create the cache - lruc *cache = (lruc *) calloc(sizeof(lruc), 1); - if(!cache) { - perror("LRU Cache unable to create cache object"); - return NULL; - } - cache->hash_table_size = cache_size / average_length; - cache->average_item_length = average_length; - cache->free_memory = cache_size; - cache->total_memory = cache_size; - cache->seed = time(NULL); - - // size the hash table to a guestimate of the number of slots required (assuming a perfect hash) - cache->items = (lruc_item **) calloc(sizeof(lruc_item *), cache->hash_table_size); - if(!cache->items) { - perror("LRU Cache unable to create cache hash table"); - free(cache); - return NULL; - } - - // all cache calls are guarded by a mutex - cache->mutex = (pthread_mutex_t *) malloc(sizeof(pthread_mutex_t)); - if(pthread_mutex_init(cache->mutex, NULL)) { - perror("LRU Cache unable to initialise mutex"); - free(cache->items); - free(cache); - return NULL; - } - return cache; -} - - -lruc_error lruc_free(lruc *cache) { - test_for_missing_cache(); - - // free each of the cached items, and the hash table - lruc_item *item = NULL, *next = NULL; - uint32_t i = 0; - if(cache->items) { - for(; i < cache->hash_table_size; i++) { - item = cache->items[i]; - while(item) { - next = (lruc_item *) item->next; - free(item); - item = next; - } - } - free(cache->items); - } - - // free the cache - if(cache->mutex) { - if(pthread_mutex_destroy(cache->mutex)) { - perror("LRU Cache unable to destroy mutex"); - return LRUC_PTHREAD_ERROR; - } - } - free(cache); - - return LRUC_NO_ERROR; -} - - -lruc_error lruc_set(lruc *cache, void *key, uint32_t key_length, void *value, uint32_t value_length) { - test_for_missing_cache(); - test_for_missing_key(); - test_for_missing_value(); - test_for_value_too_large(); - lock_cache(); - - // see if the key already exists - uint32_t hash_index = lruc_hash(cache, key, key_length), required = 0; - lruc_item *item = NULL, *prev = NULL; - item = cache->items[hash_index]; - - while(item && lruc_cmp_keys(item, key, key_length)) { - prev = item; - item = (lruc_item *) item->next; - } - - if(item) { - // update the value and value_lengths - required = value_length - item->value_length; - free(item->value); - item->value = value; - item->value_length = value_length; - - } else { - // insert a new item - item = lruc_pop_or_create_item(cache); - item->value = value; - item->key = key; - item->value_length = value_length; - item->key_length = key_length; - required = value_length; - - if(prev) - prev->next = item; - else - cache->items[hash_index] = item; - } - item->access_count = ++cache->access_count; - - // remove as many items as necessary to free enough space - if(required > 0 && required > cache->free_memory) { - while(cache->free_memory < required) - lruc_remove_lru_item(cache); - } - cache->free_memory -= required; - unlock_cache(); - return LRUC_NO_ERROR; -} - - -lruc_error lruc_get(lruc *cache, void *key, uint32_t key_length, void **value) { - test_for_missing_cache(); - test_for_missing_key(); - lock_cache(); - - // loop until we find the item, or hit the end of a chain - uint32_t hash_index = lruc_hash(cache, key, key_length); - lruc_item *item = cache->items[hash_index]; - - while(item && lruc_cmp_keys(item, key, key_length)) - item = (lruc_item *) item->next; - - if(item) { - *value = item->value; - item->access_count = ++cache->access_count; - } else { - *value = NULL; - } - - unlock_cache(); - return LRUC_NO_ERROR; -} - - -lruc_error lruc_delete(lruc *cache, void *key, uint32_t key_length) { - test_for_missing_cache(); - test_for_missing_key(); - lock_cache(); - - // loop until we find the item, or hit the end of a chain - lruc_item *item = NULL, *prev = NULL; - uint32_t hash_index = lruc_hash(cache, key, key_length); - item = cache->items[hash_index]; - - while(item && lruc_cmp_keys(item, key, key_length)) { - prev = item; - item = (lruc_item *) item->next; - } - - if(item) { - lruc_remove_item(cache, prev, item, hash_index); - } - - unlock_cache(); - return LRUC_NO_ERROR; -} -- cgit v1.2.3 From 7efe1f06eedbdfbdc316d330f746c47391be8f78 Mon Sep 17 00:00:00 2001 From: Luca Deri Date: Thu, 17 Jan 2019 22:03:39 +0100 Subject: Removed debug code --- src/lib/protocols/ssl.c | 4 ---- 1 file changed, 4 deletions(-) (limited to 'src/lib/protocols/ssl.c') diff --git a/src/lib/protocols/ssl.c b/src/lib/protocols/ssl.c index ac31fb715..4651b358f 100644 --- a/src/lib/protocols/ssl.c +++ b/src/lib/protocols/ssl.c @@ -626,10 +626,6 @@ void ndpi_search_ssl_tcp(struct ndpi_detection_module_struct *ndpi_struct, struc struct ndpi_packet_struct *packet = &flow->packet; u_int8_t ret; - if(flow->packet.tcp && (5222 == ntohs(flow->packet.tcp->dest))) - printf("%u - %u\n", ntohs(flow->packet.tcp->source), ntohs(flow->packet.tcp->dest)); - - if(packet->detected_protocol_stack[0] == NDPI_PROTOCOL_SSL) { if(flow->l4.tcp.ssl_stage == 3 && packet->payload_packet_len > 20 && flow->packet_counter < 5) { /* this should only happen, when we detected SSL with a packet that had parts of the certificate in subsequent packets -- cgit v1.2.3 From fde43804228d03911aed51076be6f4e4ab4fc0fa Mon Sep 17 00:00:00 2001 From: Luca Date: Fri, 8 Mar 2019 20:17:11 +0100 Subject: Proto cleanup --- src/include/ndpi_protocol_ids.h | 2 +- src/lib/ndpi_content_match.c.inc | 7 ------- src/lib/ndpi_main.c | 5 +++++ src/lib/protocols/ssl.c | 10 +++++----- 4 files changed, 11 insertions(+), 13 deletions(-) (limited to 'src/lib/protocols/ssl.c') diff --git a/src/include/ndpi_protocol_ids.h b/src/include/ndpi_protocol_ids.h index 1d9c1d428..7e2f55711 100644 --- a/src/include/ndpi_protocol_ids.h +++ b/src/include/ndpi_protocol_ids.h @@ -242,7 +242,7 @@ typedef enum { NDPI_PROTOCOL_GITHUB = 203, NDPI_PROTOCOL_BJNP = 204, NDPI_PROTOCOL_FREE_205 = 205, /* Free */ - NDPI_PROTOCOL_VIDTO = 206, + NDPI_PROTOCOL_FREE_206 = 206, /* Free */ NDPI_PROTOCOL_SMPP = 207, /* Damir Franusic */ NDPI_PROTOCOL_DNSCRYPT = 208, NDPI_PROTOCOL_TINC = 209, /* William Guglielmo */ diff --git a/src/lib/ndpi_content_match.c.inc b/src/lib/ndpi_content_match.c.inc index 8f7fc2337..c71913081 100644 --- a/src/lib/ndpi_content_match.c.inc +++ b/src/lib/ndpi_content_match.c.inc @@ -7947,13 +7947,6 @@ static ndpi_network host_protocol_list[] = { { 0xD040C900 /* 208.64.201.0/22 */, 22, NDPI_PROTOCOL_STEAM }, { 0xD04EA400 /* 208.78.164.0/22 */, 22, NDPI_PROTOCOL_STEAM }, - /* - VidTO - */ - { 0x51111030 /* 81.17.16.48/32 */, 32, NDPI_PROTOCOL_VIDTO }, - { 0x5fb7329d /* 95.183.50.157/32 */, 32, NDPI_PROTOCOL_VIDTO }, - { 0x577824f2 /* 87.120.36.242/32 */, 32, NDPI_PROTOCOL_VIDTO }, - { 0x0, 0, 0 } }; diff --git a/src/lib/ndpi_main.c b/src/lib/ndpi_main.c index 936f47dac..b8cdf410b 100644 --- a/src/lib/ndpi_main.c +++ b/src/lib/ndpi_main.c @@ -1263,6 +1263,11 @@ static void ndpi_init_protocol_defaults(struct ndpi_detection_module_struct *ndp no_master, "Free", NDPI_PROTOCOL_CATEGORY_CUSTOM_1 /* dummy */, ndpi_build_default_ports(ports_a, 0, 0, 0, 0, 0) /* TCP */, ndpi_build_default_ports(ports_b, 0, 0, 0, 0, 0) /* UDP */); + ndpi_set_proto_defaults(ndpi_mod, NDPI_PROTOCOL_FUN, NDPI_PROTOCOL_FREE_206, + 0 /* can_have_a_subprotocol */, no_master, + no_master, "Free", NDPI_PROTOCOL_CATEGORY_CUSTOM_1 /* dummy */, + ndpi_build_default_ports(ports_a, 0, 0, 0, 0, 0) /* TCP */, + ndpi_build_default_ports(ports_b, 0, 0, 0, 0, 0) /* UDP */); ndpi_set_proto_defaults(ndpi_mod, NDPI_PROTOCOL_FUN, NDPI_PROTOCOL_PPSTREAM, 0 /* can_have_a_subprotocol */, no_master, no_master, "PPStream", NDPI_PROTOCOL_CATEGORY_VIDEO, diff --git a/src/lib/protocols/ssl.c b/src/lib/protocols/ssl.c index 4651b358f..ff6b47a0f 100644 --- a/src/lib/protocols/ssl.c +++ b/src/lib/protocols/ssl.c @@ -27,7 +27,8 @@ #include "ndpi_api.h" -// #define CERTIFICATE_DEBUG 1 +//#define CERTIFICATE_DEBUG 1 + #define NDPI_MAX_SSL_REQUEST_SIZE 10000 /* Skype.c */ @@ -150,9 +151,9 @@ int getSSLcertificate(struct ndpi_detection_module_struct *ndpi_struct, #ifdef CERTIFICATE_DEBUG { - static u_int8_t id = 0; + u_int16_t ssl_version = (packet->payload[1] << 8) + packet->payload[2]; - NDPI_LOG_DBG2(ndpi_struct,"-> [%u] %02X\n", ++id, packet->payload[0] & 0xFF); + printf("SSL [version: %u]\n", ssl_version); } #endif @@ -232,8 +233,7 @@ int getSSLcertificate(struct ndpi_detection_module_struct *ndpi_struct, } } else if(handshake_protocol == 0x01 /* Client Hello */) { u_int offset, base_offset = 43; - if (base_offset + 2 <= packet->payload_packet_len) - { + if (base_offset + 2 <= packet->payload_packet_len) { u_int16_t session_id_len = packet->payload[base_offset]; if((session_id_len+base_offset+2) <= total_len) { -- cgit v1.2.3 From db91837647737b3d57359dd6d7bc8474644f8043 Mon Sep 17 00:00:00 2001 From: chiehminw Date: Wed, 20 Mar 2019 17:22:37 +0800 Subject: retreive ssl certificate origanization Signed-off-by: chiehminw --- src/include/ndpi_typedefs.h | 2 +- src/lib/protocols/ssl.c | 69 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 70 insertions(+), 1 deletion(-) (limited to 'src/lib/protocols/ssl.c') diff --git a/src/include/ndpi_typedefs.h b/src/include/ndpi_typedefs.h index 39995d662..0c71f9de4 100644 --- a/src/include/ndpi_typedefs.h +++ b/src/include/ndpi_typedefs.h @@ -1085,7 +1085,7 @@ struct ndpi_flow_struct { struct { struct { - char client_certificate[64], server_certificate[64]; + char client_certificate[64], server_certificate[64], server_organization[64]; } ssl; struct { diff --git a/src/lib/protocols/ssl.c b/src/lib/protocols/ssl.c index ff6b47a0f..845f20de6 100644 --- a/src/lib/protocols/ssl.c +++ b/src/lib/protocols/ssl.c @@ -335,18 +335,87 @@ int getSSLcertificate(struct ndpi_detection_module_struct *ndpi_struct, return(0); /* Not found */ } +void getSSLorganization(struct ndpi_detection_module_struct *ndpi_struct, + struct ndpi_flow_struct *flow, + char *buffer, int buffer_len) { + struct ndpi_packet_struct *packet = &flow->packet; + + if(packet->payload[0] != 0x16 /* Handshake */) + return; + + u_int16_t total_len = (packet->payload[3] << 8) + packet->payload[4] + 5 /* SSL Header */; + u_int8_t handshake_protocol = packet->payload[5]; /* handshake protocol a bit misleading, it is message type according TLS specs */ + + if(handshake_protocol != 0x02 && handshake_protocol != 0xb /* Server Hello and Certificate message types are interesting for us */) + return; + + /* Truncate total len, search at least in incomplete packet */ + if(total_len > packet->payload_packet_len) + total_len = packet->payload_packet_len; + + memset(buffer, 0, buffer_len); + + /* Check after handshake protocol header (5 bytes) and message header (4 bytes) */ + u_int num_found = 0; + u_int i, j; + for(i = 9; i < packet->payload_packet_len-4; i++) { + /* Organization OID: 2.5.4.10 */ + if((packet->payload[i] == 0x55) && (packet->payload[i+1] == 0x04) && (packet->payload[i+2] == 0x0a)) { + u_int8_t type_tag = packet->payload[i+3]; // 0x0c: utf8string / 0x13: printable_string + u_int8_t server_len = packet->payload[i+4]; + + num_found++; + /* what we want is subject certificate, so we bypass the issuer certificate */ + if (num_found != 2) continue; + + // packet is truncated... further inspection is not needed + if(i+4+server_len >= packet->payload_packet_len) { + break; + } + + char *server_org = (char*)&packet->payload[i+5]; + + u_int len = (u_int)ndpi_min(server_len, buffer_len-1); + strncpy(buffer, server_org, len); + buffer[len] = '\0'; + + // check if organization string are all printable + u_int8_t is_printable = 1; + for (j = 0; j < len; j++) { + if(!ndpi_isprint(buffer[j])) { + is_printable = 0; + break; + } + } + + if (is_printable == 1) { + snprintf(flow->protos.stun_ssl.ssl.server_organization, + sizeof(flow->protos.stun_ssl.ssl.server_organization), "%s", buffer); +#ifdef CERTIFICATE_DEBUG + printf("Certificate origanization: %s\n", flow->protos.stun_ssl.ssl.server_organization); +#endif + } + } + } +} + int sslTryAndRetrieveServerCertificate(struct ndpi_detection_module_struct *ndpi_struct, struct ndpi_flow_struct *flow) { struct ndpi_packet_struct *packet = &flow->packet; /* consider only specific SSL packets (handshake) */ if((packet->payload_packet_len > 9) && (packet->payload[0] == 0x16)) { char certificate[64]; + char organization[64]; int rc; certificate[0] = '\0'; rc = getSSLcertificate(ndpi_struct, flow, certificate, sizeof(certificate)); packet->ssl_certificate_num_checks++; if (rc > 0) { + // try fetch server organization once server certificate is found + organization[0] = '\0'; + getSSLorganization(ndpi_struct, flow, organization, sizeof(organization)); + packet->ssl_certificate_detected++; if ((flow->l4.tcp.ssl_seen_server_cert == 1) && (flow->protos.stun_ssl.ssl.server_certificate[0] != '\0')) /* 0 means we're done processing extra packets (since we found what we wanted) */ -- cgit v1.2.3 From 96ed31874dd62caef9e6f4dfe5c3825e88151c20 Mon Sep 17 00:00:00 2001 From: chiehminw Date: Wed, 20 Mar 2019 17:25:39 +0800 Subject: we only need one dot to confirm it is a valid domain name github.com is valid but can not be retreive by server certificate parsing Signed-off-by: chiehminw --- src/lib/protocols/ssl.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src/lib/protocols/ssl.c') diff --git a/src/lib/protocols/ssl.c b/src/lib/protocols/ssl.c index 845f20de6..e6cc48bba 100644 --- a/src/lib/protocols/ssl.c +++ b/src/lib/protocols/ssl.c @@ -216,11 +216,11 @@ int getSSLcertificate(struct ndpi_detection_module_struct *ndpi_struct, break; } else if(buffer[j] == '.') { num_dots++; - if(num_dots >=2) break; + if(num_dots >=1) break; } } - if(num_dots >= 2) { + if(num_dots >= 1) { if(!ndpi_struct->disable_metadata_export) { stripCertificateTrailer(buffer, buffer_len); snprintf(flow->protos.stun_ssl.ssl.server_certificate, -- cgit v1.2.3 From 0b5e5a5f82ce87c14a5ecd588ed1dd0b08ff5e18 Mon Sep 17 00:00:00 2001 From: Luca Deri Date: Mon, 25 Mar 2019 22:08:39 +0100 Subject: SSL cleanup --- src/include/ndpi_typedefs.h | 5 ++- src/lib/protocols/ssl.c | 89 +++++++++++++++++++++++++-------------------- 2 files changed, 53 insertions(+), 41 deletions(-) (limited to 'src/lib/protocols/ssl.c') diff --git a/src/include/ndpi_typedefs.h b/src/include/ndpi_typedefs.h index c49768f64..92c011b3d 100644 --- a/src/include/ndpi_typedefs.h +++ b/src/include/ndpi_typedefs.h @@ -541,7 +541,10 @@ struct ndpi_flow_tcp_struct { u_int32_t telnet_stage:2; // 0 - 2 /* NDPI_PROTOCOL_SSL */ - u_int8_t ssl_seen_client_cert:1, ssl_seen_server_cert:1, ssl_stage:2; // 0 - 5 + u_int8_t ssl_seen_client_cert:1, + ssl_seen_server_cert:1, + ssl_seen_certificate:1, + ssl_stage:2; // 0 - 5 /* NDPI_PROTOCOL_POSTGRES */ u_int32_t postgres_stage:3; diff --git a/src/lib/protocols/ssl.c b/src/lib/protocols/ssl.c index e6cc48bba..05988a8d4 100644 --- a/src/lib/protocols/ssl.c +++ b/src/lib/protocols/ssl.c @@ -27,7 +27,7 @@ #include "ndpi_api.h" -//#define CERTIFICATE_DEBUG 1 +// #define CERTIFICATE_DEBUG 1 #define NDPI_MAX_SSL_REQUEST_SIZE 10000 @@ -152,8 +152,9 @@ int getSSLcertificate(struct ndpi_detection_module_struct *ndpi_struct, #ifdef CERTIFICATE_DEBUG { u_int16_t ssl_version = (packet->payload[1] << 8) + packet->payload[2]; - - printf("SSL [version: %u]\n", ssl_version); + u_int16_t ssl_len = (packet->payload[3] << 8) + packet->payload[4]; + + printf("SSL Record [version: 0x%02X][len: %u]\n", ssl_version, ssl_len); } #endif @@ -175,10 +176,18 @@ int getSSLcertificate(struct ndpi_detection_module_struct *ndpi_struct, if(total_len > 4) { int i; - if(handshake_protocol == 0x02 || handshake_protocol == 0xb /* Server Hello and Certificate message types are interesting for us */) { +#ifdef CERTIFICATE_DEBUG + printf("SSL [len: %u][handshake_protocol: %02X]\n", packet->payload_packet_len, handshake_protocol); +#endif + + if((handshake_protocol == 0x02) + || (handshake_protocol == 0xb) /* Server Hello and Certificate message types are interesting for us */) { u_int num_found = 0; - flow->l4.tcp.ssl_seen_server_cert = 1; + if(handshake_protocol == 0x02) + flow->l4.tcp.ssl_seen_server_cert = 1; + else + flow->l4.tcp.ssl_seen_certificate = 1; /* Check after handshake protocol header (5 bytes) and message header (4 bytes) */ for(i = 9; i < packet->payload_packet_len-3; i++) { @@ -233,7 +242,7 @@ int getSSLcertificate(struct ndpi_detection_module_struct *ndpi_struct, } } else if(handshake_protocol == 0x01 /* Client Hello */) { u_int offset, base_offset = 43; - if (base_offset + 2 <= packet->payload_packet_len) { + if(base_offset + 2 <= packet->payload_packet_len) { u_int16_t session_id_len = packet->payload[base_offset]; if((session_id_len+base_offset+2) <= total_len) { @@ -366,7 +375,7 @@ void getSSLorganization(struct ndpi_detection_module_struct *ndpi_struct, num_found++; /* what we want is subject certificate, so we bypass the issuer certificate */ - if (num_found != 2) continue; + if(num_found != 2) continue; // packet is truncated... further inspection is not needed if(i+4+server_len >= packet->payload_packet_len) { @@ -388,7 +397,7 @@ void getSSLorganization(struct ndpi_detection_module_struct *ndpi_struct, } } - if (is_printable == 1) { + if(is_printable == 1) { snprintf(flow->protos.stun_ssl.ssl.server_organization, sizeof(flow->protos.stun_ssl.ssl.server_organization), "%s", buffer); #ifdef CERTIFICATE_DEBUG @@ -411,18 +420,18 @@ int sslTryAndRetrieveServerCertificate(struct ndpi_detection_module_struct *ndpi certificate[0] = '\0'; rc = getSSLcertificate(ndpi_struct, flow, certificate, sizeof(certificate)); packet->ssl_certificate_num_checks++; - if (rc > 0) { + if(rc > 0) { // try fetch server organization once server certificate is found organization[0] = '\0'; getSSLorganization(ndpi_struct, flow, organization, sizeof(organization)); packet->ssl_certificate_detected++; - if ((flow->l4.tcp.ssl_seen_server_cert == 1) && (flow->protos.stun_ssl.ssl.server_certificate[0] != '\0')) + if((flow->l4.tcp.ssl_seen_server_cert == 1) && (flow->protos.stun_ssl.ssl.server_certificate[0] != '\0')) /* 0 means we're done processing extra packets (since we found what we wanted) */ return 0; } /* Client hello, Server Hello, and certificate packets probably all checked in this case */ - if ((packet->ssl_certificate_num_checks >= 3) + if((packet->ssl_certificate_num_checks >= 3) && (flow->l4.tcp.seen_syn) && (flow->l4.tcp.seen_syn_ack) && (flow->l4.tcp.seen_ack) /* We have seen the 3-way handshake */) @@ -438,7 +447,7 @@ int sslTryAndRetrieveServerCertificate(struct ndpi_detection_module_struct *ndpi void sslInitExtraPacketProcessing(int caseNum, struct ndpi_flow_struct *flow) { flow->check_extra_packets = 1; /* 0 is the case for waiting for the server certificate */ - if (caseNum == 0) { + if(caseNum == 0) { /* At most 7 packets should almost always be enough to find the server certificate if it's there */ flow->max_extra_packets_to_check = 7; flow->extra_packets_func = sslTryAndRetrieveServerCertificate; @@ -474,7 +483,8 @@ int sslDetectProtocolFromCertificate(struct ndpi_detection_module_struct *ndpi_s /* If we've detected the subprotocol from client certificate but haven't had a chance * to see the server certificate yet, set up extra packet processing to wait * a few more packets. */ - if(((flow->l4.tcp.ssl_seen_client_cert == 1) && (flow->protos.stun_ssl.ssl.client_certificate[0] != '\0')) && ((flow->l4.tcp.ssl_seen_server_cert != 1) && (flow->protos.stun_ssl.ssl.server_certificate[0] == '\0'))) { + if(((flow->l4.tcp.ssl_seen_client_cert == 1) && (flow->protos.stun_ssl.ssl.client_certificate[0] != '\0')) + && ((flow->l4.tcp.ssl_seen_server_cert != 1) && (flow->protos.stun_ssl.ssl.server_certificate[0] == '\0'))) { sslInitExtraPacketProcessing(0, flow); } @@ -487,11 +497,13 @@ int sslDetectProtocolFromCertificate(struct ndpi_detection_module_struct *ndpi_s return(rc); } - if(((packet->ssl_certificate_num_checks >= 2) + if(((packet->ssl_certificate_num_checks >= 3) && flow->l4.tcp.seen_syn && flow->l4.tcp.seen_syn_ack && flow->l4.tcp.seen_ack /* We have seen the 3-way handshake */) - || ((flow->l4.tcp.ssl_seen_server_cert == 1) && (flow->protos.stun_ssl.ssl.server_certificate[0] != '\0')) + || ((flow->l4.tcp.ssl_seen_certificate == 1) + && (flow->l4.tcp.ssl_seen_server_cert == 1) + && (flow->protos.stun_ssl.ssl.server_certificate[0] != '\0')) /* || ((flow->l4.tcp.ssl_seen_client_cert == 1) && (flow->protos.stun_ssl.ssl.client_certificate[0] != '\0')) */ ) { ndpi_int_ssl_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_SSL); @@ -501,8 +513,7 @@ int sslDetectProtocolFromCertificate(struct ndpi_detection_module_struct *ndpi_s return(0); } -static void ssl_mark_and_payload_search_for_other_protocols(struct - ndpi_detection_module_struct +static void ssl_mark_and_payload_search_for_other_protocols(struct ndpi_detection_module_struct *ndpi_struct, struct ndpi_flow_struct *flow) { struct ndpi_packet_struct *packet = &flow->packet; @@ -579,16 +590,16 @@ static void ssl_mark_and_payload_search_for_other_protocols(struct /* SSL without certificate (Skype, Ultrasurf?) */ NDPI_LOG_INFO(ndpi_struct, "found ssl NO_CERT\n"); ndpi_int_ssl_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_SSL_NO_CERT); - } else + } else if(packet->ssl_certificate_num_checks >= 3) { NDPI_LOG_INFO(ndpi_struct, "found ssl\n"); - ndpi_int_ssl_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_SSL); + ndpi_int_ssl_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_SSL); + } } } static u_int8_t ndpi_search_sslv3_direction1(struct ndpi_detection_module_struct *ndpi_struct, struct ndpi_flow_struct *flow) { - struct ndpi_packet_struct *packet = &flow->packet; if((packet->payload_packet_len >= 5) @@ -716,27 +727,25 @@ void ndpi_search_ssl_tcp(struct ndpi_detection_module_struct *ndpi_struct, struc NDPI_LOG_DBG(ndpi_struct, "search ssl\n"); - { - /* Check if this is whatsapp first (this proto runs over port 443) */ - if((packet->payload_packet_len > 5) - && ((packet->payload[0] == 'W') - && (packet->payload[1] == 'A') - && (packet->payload[4] == 0) - && (packet->payload[2] <= 9) - && (packet->payload[3] <= 9))) { - ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_WHATSAPP, NDPI_PROTOCOL_UNKNOWN); - return; - } else if((packet->payload_packet_len == 4) - && (packet->payload[0] == 'W') - && (packet->payload[1] == 'A')) { - ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_WHATSAPP, NDPI_PROTOCOL_UNKNOWN); + /* Check if this is whatsapp first (this proto runs over port 443) */ + if((packet->payload_packet_len > 5) + && ((packet->payload[0] == 'W') + && (packet->payload[1] == 'A') + && (packet->payload[4] == 0) + && (packet->payload[2] <= 9) + && (packet->payload[3] <= 9))) { + ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_WHATSAPP, NDPI_PROTOCOL_UNKNOWN); + return; + } else if((packet->payload_packet_len == 4) + && (packet->payload[0] == 'W') + && (packet->payload[1] == 'A')) { + ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_WHATSAPP, NDPI_PROTOCOL_UNKNOWN); + return; + } else { + /* No whatsapp, let's try SSL */ + if(sslDetectProtocolFromCertificate(ndpi_struct, flow) > 0) return; - } else { - /* No whatsapp, let's try SSL */ - if(sslDetectProtocolFromCertificate(ndpi_struct, flow) > 0) - return; - } - } + } if(packet->payload_packet_len > 40 && flow->l4.tcp.ssl_stage == 0) { NDPI_LOG_DBG2(ndpi_struct, "first ssl packet\n"); -- cgit v1.2.3