From b1282da5424cfc7fbe331ac5256148780a5fa2fb Mon Sep 17 00:00:00 2001
From: Guido Falsi <mad@madpilot.net>
Date: Tue, 12 Mar 2019 18:12:18 +0100
Subject: The le32toh() function used in some places on BSD OSes require
 sys/endian.h to be included.

---
 src/lib/ndpi_main.c | 4 ++++
 1 file changed, 4 insertions(+)

(limited to 'src/lib/ndpi_main.c')

diff --git a/src/lib/ndpi_main.c b/src/lib/ndpi_main.c
index 2471a0eb1..4e9f37248 100644
--- a/src/lib/ndpi_main.c
+++ b/src/lib/ndpi_main.c
@@ -41,6 +41,10 @@
 #include <unistd.h>
 #endif
 
+#if defined __FreeBSD__ || defined __NetBSD__ || defined __OpenBSD__
+#include <sys/endian.h>
+#endif
+
 #include "ndpi_content_match.c.inc"
 #include "third_party/include/ndpi_patricia.h"
 #include "third_party/include/ht_hash.h"
-- 
cgit v1.2.3


From f47be6ef6045a97a20f7a929d15a0354260c0414 Mon Sep 17 00:00:00 2001
From: Luca Deri <deri@ntop.org>
Date: Sat, 23 Mar 2019 10:32:18 +0100
Subject: Added SSL organization support

---
 example/ndpiReader.c | 1 +
 example/ndpi_util.c  | 2 ++
 example/ndpi_util.h  | 2 +-
 src/lib/ndpi_main.c  | 2 +-
 4 files changed, 5 insertions(+), 2 deletions(-)

(limited to 'src/lib/ndpi_main.c')

diff --git a/example/ndpiReader.c b/example/ndpiReader.c
index fe0ea879b..4244e3745 100644
--- a/example/ndpiReader.c
+++ b/example/ndpiReader.c
@@ -804,6 +804,7 @@ static void printFlow(u_int16_t id, struct ndpi_flow_info *flow, u_int16_t threa
 
     if(flow->ssh_ssl.client_info[0] != '\0') fprintf(out, "[client: %s]", flow->ssh_ssl.client_info);
     if(flow->ssh_ssl.server_info[0] != '\0') fprintf(out, "[server: %s]", flow->ssh_ssl.server_info);
+    if(flow->ssh_ssl.server_organization[0] != '\0') fprintf(out, "[organization: %s]", flow->ssh_ssl.server_organization);
     if(flow->bittorent_hash[0] != '\0') fprintf(out, "[BT Hash: %s]", flow->bittorent_hash);
 
     fprintf(out, "\n");
diff --git a/example/ndpi_util.c b/example/ndpi_util.c
index 3a2a66c97..9e1e72132 100644
--- a/example/ndpi_util.c
+++ b/example/ndpi_util.c
@@ -566,6 +566,8 @@ void process_ndpi_collected_info(struct ndpi_workflow * workflow, struct ndpi_fl
 	       flow->ndpi_flow->protos.stun_ssl.ssl.client_certificate);
       snprintf(flow->ssh_ssl.server_info, sizeof(flow->ssh_ssl.server_info), "%s",
 	       flow->ndpi_flow->protos.stun_ssl.ssl.server_certificate);
+      snprintf(flow->ssh_ssl.server_organization, sizeof(flow->ssh_ssl.server_organization), "%s",
+	       flow->ndpi_flow->protos.stun_ssl.ssl.server_organization);
     }
   }
 
diff --git a/example/ndpi_util.h b/example/ndpi_util.h
index eb9ab8e65..0a5a3b8c2 100644
--- a/example/ndpi_util.h
+++ b/example/ndpi_util.h
@@ -97,7 +97,7 @@ typedef struct ndpi_flow_info {
   char bittorent_hash[41];
   
   struct {
-    char client_info[64], server_info[64];
+    char client_info[64], server_info[64], server_organization[64];
   } ssh_ssl;
 
   void *src_id, *dst_id;
diff --git a/src/lib/ndpi_main.c b/src/lib/ndpi_main.c
index 047167ac3..59de7a763 100644
--- a/src/lib/ndpi_main.c
+++ b/src/lib/ndpi_main.c
@@ -1,7 +1,7 @@
 /*
  * ndpi_main.c
  *
- * Copyright (C) 2011-18 - ntop.org
+ * Copyright (C) 2011-19 - ntop.org
  *
  * This file is part of nDPI, an open source deep packet inspection
  * library based on the OpenDPI and PACE technology by ipoque GmbH
-- 
cgit v1.2.3


From 1915a63cf29fbe3d2b0a983b2875929518d242ad Mon Sep 17 00:00:00 2001
From: Luca Deri <deri@ntop.org>
Date: Mon, 1 Apr 2019 12:30:51 +0200
Subject: Implemented ndpi_process_partial_detection() API call to handle
 partial matches due to the nDPI specified configuration

---
 example/ndpiReader.c     |  31 ++++-----
 src/include/ndpi_api.h   |  15 +++-
 src/lib/ndpi_main.c      | 176 ++++++++++++++++++++++++++++++-----------------
 src/lib/protocols/dns.c  |   6 +-
 src/lib/protocols/http.c |   4 +-
 5 files changed, 149 insertions(+), 83 deletions(-)

(limited to 'src/lib/ndpi_main.c')

diff --git a/example/ndpiReader.c b/example/ndpiReader.c
index 4244e3745..72eaecca4 100644
--- a/example/ndpiReader.c
+++ b/example/ndpiReader.c
@@ -914,11 +914,10 @@ static void node_proto_guess_walker(const void *node, ndpi_VISIT which, int dept
   struct ndpi_flow_info *flow = *(struct ndpi_flow_info **) node;
   u_int16_t thread_id = *((u_int16_t *) user_data);
 
-  if((which == ndpi_preorder) || (which == ndpi_leaf)) { /* Avoid walking the same node multiple times */    
-    if((!flow->detection_completed) && flow->ndpi_flow) {   
+  if((which == ndpi_preorder) || (which == ndpi_leaf)) { /* Avoid walking the same node multiple times */
+    if((!flow->detection_completed) && flow->ndpi_flow)
       flow->detected_protocol = ndpi_detection_giveup(ndpi_thread_info[0].workflow->ndpi_struct, flow->ndpi_flow, enable_protocol_guess);
-    }
-    
+
     process_ndpi_collected_info(ndpi_thread_info[thread_id].workflow, flow);
 
     ndpi_thread_info[thread_id].workflow->stats.protocol_counter[flow->detected_protocol.app_protocol]       += flow->src2dst_packets + flow->dst2src_packets;
@@ -1956,7 +1955,7 @@ static void printResults(u_int64_t processing_time_usec, u_int64_t setup_time_us
     printf("\tPeak Memory:             %-13s\n", formatBytes(max_ndpi_memory, buf, sizeof(buf)));
     printf("\tSetup Time:              %lu msec\n", (unsigned long)(setup_time_usec/1000));
     printf("\tPacket Processing Time:  %lu msec\n", (unsigned long)(processing_time_usec/1000));
-    
+
     if(!json_flag) {
       printf("\nTraffic statistics:\n");
       printf("\tEthernet bytes:        %-13llu (includes ethernet CRC/IFC/trailer)\n",
@@ -1992,10 +1991,10 @@ static void printResults(u_int64_t processing_time_usec, u_int64_t setup_time_us
 	float t = (float)(cumulative_stats.ip_packet_count*1000000)/(float)processing_time_usec;
 	float b = (float)(cumulative_stats.total_wire_bytes * 8 *1000000)/(float)processing_time_usec;
 	float traffic_duration;
-	
+
 	if(live_capture) traffic_duration = processing_time_usec;
 	else traffic_duration = (pcap_end.tv_sec*1000000 + pcap_end.tv_usec) - (pcap_start.tv_sec*1000000 + pcap_start.tv_usec);
-	
+
 	printf("\tnDPI throughput:       %s pps / %s/sec\n", formatPackets(t, buf), formatTraffic(b, 1, buf1));
 	t = (float)(cumulative_stats.ip_packet_count*1000000)/(float)traffic_duration;
 	b = (float)(cumulative_stats.total_wire_bytes * 8 *1000000)/(float)traffic_duration;
@@ -2005,7 +2004,7 @@ static void printResults(u_int64_t processing_time_usec, u_int64_t setup_time_us
 	strftime(when, sizeof(when), "%d/%b/%Y %H:%M:%S", localtime(&pcap_end.tv_sec));
 	printf("\tAnalysis end:          %s\n", when);
 	printf("\tTraffic throughput:    %s pps / %s/sec\n", formatPackets(t, buf), formatTraffic(b, 1, buf1));
-	printf("\tTraffic duration:      %.3f sec\n", traffic_duration/1000000);	
+	printf("\tTraffic duration:      %.3f sec\n", traffic_duration/1000000);
       }
 
       if(enable_protocol_guess)
@@ -2490,7 +2489,7 @@ static void ndpi_process_packet(u_char *args,
   if(memcmp(packet, packet_checked, header->caplen) != 0)
     printf("INTERNAL ERROR: ingress packet was modified by nDPI: this should not happen [thread_id=%u, packetId=%lu, caplen=%u]\n",
 	   thread_id, (unsigned long)ndpi_thread_info[thread_id].workflow->stats.raw_packet_count, header->caplen);
-  
+
   if((pcap_end.tv_sec-pcap_start.tv_sec) > pcap_analysis_duration) {
     int i;
     u_int64_t processing_time_usec, setup_time_usec;
@@ -2498,7 +2497,7 @@ static void ndpi_process_packet(u_char *args,
     gettimeofday(&end, NULL);
     processing_time_usec = end.tv_sec*1000000 + end.tv_usec - (begin.tv_sec*1000000 + begin.tv_usec);
     setup_time_usec = begin.tv_sec*1000000 + begin.tv_usec - (startup_time.tv_sec*1000000 + startup_time.tv_usec);
-    
+
     printResults(processing_time_usec, setup_time_usec);
 
     for(i=0; i<ndpi_thread_info[thread_id].workflow->prefs.num_roots; i++) {
@@ -2515,7 +2514,7 @@ static void ndpi_process_packet(u_char *args,
     memcpy(&pcap_start, &pcap_end, sizeof(pcap_start));
   }
 
-  /* 
+  /*
      Leave the free as last statement to avoid crashes when ndpi_detection_giveup()
      is called above by printResults()
   */
@@ -2541,7 +2540,7 @@ void * processing_thread(void *_thread_id) {
 #if defined(linux) && defined(HAVE_PTHREAD_SETAFFINITY_NP)
   if(core_affinity[thread_id] >= 0) {
     cpu_set_t cpuset;
-    
+
     CPU_ZERO(&cpuset);
     CPU_SET(core_affinity[thread_id], &cpuset);
 
@@ -2559,7 +2558,7 @@ void * processing_thread(void *_thread_id) {
     struct rte_mbuf *bufs[BURST_SIZE];
     u_int16_t num = rte_eth_rx_burst(dpdk_port_id, 0, bufs, BURST_SIZE);
     u_int i;
-    
+
     if(num == 0) {
       usleep(1);
       continue;
@@ -3273,8 +3272,8 @@ int orginal_main(int argc, char **argv) {
 #else
   int main(int argc, char **argv) {
 #endif
-    int i;    
-    
+    int i;
+
     if(ndpi_get_api_version() != NDPI_API_VERSION) {
       printf("nDPI Library version mismatch: please make sure this code and the nDPI library are in sync\n");
       return(-1);
@@ -3284,7 +3283,7 @@ int orginal_main(int argc, char **argv) {
 
     gettimeofday(&startup_time, NULL);
     ndpi_info_mod = ndpi_init_detection_module();
-	   
+
     if(ndpi_info_mod == NULL) return -1;
 
     memset(ndpi_thread_info, 0, sizeof(ndpi_thread_info));
diff --git a/src/include/ndpi_api.h b/src/include/ndpi_api.h
index 386d306fd..84633c3ae 100644
--- a/src/include/ndpi_api.h
+++ b/src/include/ndpi_api.h
@@ -1,7 +1,7 @@
 /*
  * ndpi_api.h
  *
- * Copyright (C) 2011-17 - ntop.org
+ * Copyright (C) 2011-19 - ntop.org
  *
  * This file is part of nDPI, an open source deep packet inspection
  * library based on the OpenDPI and PACE technology by ipoque GmbH
@@ -206,7 +206,18 @@ extern "C" {
    */
   void ndpi_set_protocol_detection_bitmask2(struct ndpi_detection_module_struct *ndpi_struct,
 					    const NDPI_PROTOCOL_BITMASK * detection_bitmask);
-
+  
+  /**
+   *  Function to be called to see in case of unknown match to see if there is
+   *  a partial match that has been prevented by the current nDPI preferences configuration
+   *
+   * @par    ndpi_struct  = the detection module
+   * @par    flow         = the flow given for the detection module
+   * @return the detected protocol even if the flow is not completed;
+   *
+   */
+  ndpi_protocol ndpi_get_partial_detection(struct ndpi_detection_module_struct *ndpi_struct,
+					   struct ndpi_flow_struct *flow);
   /**
    *  Function to be called before we give up with detection for a given flow.
    *  This function reduces the NDPI_UNKNOWN_PROTOCOL detection
diff --git a/src/lib/ndpi_main.c b/src/lib/ndpi_main.c
index 59de7a763..acbe5b8ec 100644
--- a/src/lib/ndpi_main.c
+++ b/src/lib/ndpi_main.c
@@ -669,7 +669,7 @@ void ndpi_set_proto_defaults(struct ndpi_detection_module_struct *ndpi_mod,
     ndpi_mod->proto_defaults[protoId].protoId = protoId,
     ndpi_mod->proto_defaults[protoId].protoBreed = breed;
   ndpi_mod->proto_defaults[protoId].can_have_a_subprotocol = can_have_a_subprotocol;
-  
+
   memcpy(&ndpi_mod->proto_defaults[protoId].master_tcp_protoId, tcp_master_protoId, 2*sizeof(u_int16_t));
   memcpy(&ndpi_mod->proto_defaults[protoId].master_udp_protoId, udp_master_protoId, 2*sizeof(u_int16_t));
 
@@ -677,7 +677,7 @@ void ndpi_set_proto_defaults(struct ndpi_detection_module_struct *ndpi_mod,
     if(udpDefPorts[j].port_low != 0)
       addDefaultPort(ndpi_mod, &udpDefPorts[j],
 		     &ndpi_mod->proto_defaults[protoId], 0, &ndpi_mod->udpRoot, __FUNCTION__,__LINE__);
-    
+
     if(tcpDefPorts[j].port_low != 0)
       addDefaultPort(ndpi_mod, &tcpDefPorts[j],
 		     &ndpi_mod->proto_defaults[protoId], 0, &ndpi_mod->tcpRoot, __FUNCTION__,__LINE__);
@@ -790,15 +790,15 @@ static int ndpi_string_to_automa(struct ndpi_detection_module_struct *ndpi_struc
 
   if(automa->ac_automa == NULL) return(-2);
   ac_pattern.astring = value,
-    ac_pattern.rep.number = protocol_id,  
+    ac_pattern.rep.number = protocol_id,
     ac_pattern.rep.category = (u_int16_t)category,
     ac_pattern.rep.breed = (u_int16_t)breed;
-  
+
 #ifdef MATCH_DEBUG
   printf("Adding to automa [%s][protocol_id: %u][category: %u][breed: %u]\n",
 	 value, protocol_id, category, breed);
 #endif
-  
+
   if(value == NULL)
     ac_pattern.length = 0;
   else
@@ -856,14 +856,14 @@ void ndpi_init_protocol_match(struct ndpi_detection_module_struct *ndpi_mod,
   u_int16_t no_master[2] = { NDPI_PROTOCOL_NO_MASTER_PROTO, NDPI_PROTOCOL_NO_MASTER_PROTO };
   ndpi_port_range ports_a[MAX_DEFAULT_PORTS], ports_b[MAX_DEFAULT_PORTS];
   static u_int16_t generic_id = NDPI_LAST_IMPLEMENTED_PROTOCOL;
-  
+
   if(ndpi_mod->proto_defaults[match->protocol_id].protoName == NULL) {
     if(match->protocol_id == NDPI_PROTOCOL_GENERIC)
       ndpi_mod->proto_defaults[match->protocol_id].protoName   = ndpi_strdup(NDPI_CONST_GENERIC_PROTOCOL_NAME);
    else
      ndpi_mod->proto_defaults[match->protocol_id].protoName    = ndpi_strdup(match->proto_name);
-    
-    ndpi_mod->proto_defaults[match->protocol_id].protoId       = match->protocol_id;  
+
+    ndpi_mod->proto_defaults[match->protocol_id].protoId       = match->protocol_id;
     ndpi_mod->proto_defaults[match->protocol_id].protoCategory = match->protocol_category;
     ndpi_mod->proto_defaults[match->protocol_id].protoBreed    = match->protocol_breed;
 
@@ -875,7 +875,7 @@ void ndpi_init_protocol_match(struct ndpi_detection_module_struct *ndpi_mod,
 			    ndpi_mod->proto_defaults[match->protocol_id].protoName,
 			    ndpi_mod->proto_defaults[match->protocol_id].protoCategory,
 			    ndpi_build_default_ports(ports_a, 0, 0, 0, 0, 0) /* TCP */,
-			    ndpi_build_default_ports(ports_b, 0, 0, 0, 0, 0) /* UDP */);    
+			    ndpi_build_default_ports(ports_b, 0, 0, 0, 0, 0) /* UDP */);
   }
 
   ndpi_add_host_url_subprotocol(ndpi_mod,
@@ -1025,7 +1025,7 @@ static void init_string_based_protocols(struct ndpi_detection_module_struct *ndp
   for(i=0; host_match[i].string_to_match != NULL; i++)
     ndpi_init_protocol_match(ndpi_mod, &host_match[i]);
 
-#ifdef MATCH_DEBUG  
+#ifdef MATCH_DEBUG
   // ac_automata_display(ndpi_mod->host_automa.ac_automa, 'n');
 #endif
 
@@ -1065,7 +1065,7 @@ int ndpi_set_detection_preferences(struct ndpi_detection_module_struct *ndpi_mod
   case ndpi_pref_enable_category_substring_match:
     ndpi_mod->enable_category_substring_match = (u_int8_t)value;
     break;
-    
+
   default:
     return(-1);
   }
@@ -2089,7 +2089,7 @@ static int ac_match_handler(AC_MATCH_t *m, AC_TEXT_t *txt, AC_REP_t *match) {
   char buf[64] = { '\0' };
   int min_buf_len = (txt->length > 63 /* sizeof(buf)-1 */) ? 63 : txt->length;
   u_int buf_len = strlen(buf);
-  
+
   strncpy(buf, txt->astring, min_buf_len);
   buf[min_buf_len] = '\0';
 
@@ -2103,8 +2103,8 @@ static int ac_match_handler(AC_MATCH_t *m, AC_TEXT_t *txt, AC_REP_t *match) {
     char *whatfound = strstr(buf, m->patterns->astring);
 
 #ifdef MATCH_DEBUG
-    printf("[NDPI] %s() [searching=%s][pattern=%s][%s][%c]\n", 
-	   __FUNCTION__, buf,  m->patterns->astring, 
+    printf("[NDPI] %s() [searching=%s][pattern=%s][%s][%c]\n",
+	   __FUNCTION__, buf,  m->patterns->astring,
 	   whatfound ? whatfound : "<NULL>",
 	   whatfound[-1]);
 #endif
@@ -2117,7 +2117,7 @@ static int ac_match_handler(AC_MATCH_t *m, AC_TEXT_t *txt, AC_REP_t *match) {
     if(whatfound && (whatfound != buf)
        && (m->patterns->astring[0] != '.')  /* The searched patter does not start with . */
        && strchr(m->patterns->astring, '.') /* The matched pattern has a . (e.g. numeric or sym IPs) */
-       && (whatfound[-1] != '.') 
+       && (whatfound[-1] != '.')
        )
       return(0);
   }
@@ -2368,7 +2368,7 @@ struct ndpi_detection_module_struct *ndpi_init_detection_module(void) {
 #endif
 
   ndpi_str->custom_categories.hostnames_hash            = NULL;
-  
+
   ndpi_str->custom_categories.ipAddresses                = ndpi_New_Patricia(32 /* IPv4 */);
   ndpi_str->custom_categories.ipAddresses_shadow         = ndpi_New_Patricia(32 /* IPv4 */);
 
@@ -2436,7 +2436,7 @@ int ndpi_match_string(void *_automa, char *string_to_match) {
 int ndpi_match_string_id(void *_automa, char *string_to_match, unsigned long *id) {
   AC_TEXT_t ac_input_text;
   AC_AUTOMATA_t *automa = (AC_AUTOMATA_t*)_automa;
-  AC_REP_t match = { NDPI_PROTOCOL_UNKNOWN, NDPI_PROTOCOL_CATEGORY_UNSPECIFIED, NDPI_PROTOCOL_UNRATED }; 
+  AC_REP_t match = { NDPI_PROTOCOL_UNKNOWN, NDPI_PROTOCOL_CATEGORY_UNSPECIFIED, NDPI_PROTOCOL_UNRATED };
 
   *id = -1;
   if((automa == NULL)
@@ -2449,7 +2449,7 @@ int ndpi_match_string_id(void *_automa, char *string_to_match, unsigned long *id
   ac_automata_reset(automa);
 
   *id = match.number;
-  
+
   return(*id != NDPI_PROTOCOL_UNKNOWN ? 0 : -1);
 }
 
@@ -2476,7 +2476,7 @@ static int hyperscanCustomEventHandler(unsigned int id,
 int ndpi_match_custom_category(struct ndpi_detection_module_struct *ndpi_struct,
 				      char *name, unsigned long *id) {
 #ifdef DEBUG
-  printf("[NDPI] %s(%s) [enable_category_substring_match: %u]\n", 
+  printf("[NDPI] %s(%s) [enable_category_substring_match: %u]\n",
 	 __FUNCTION__, name, ndpi_struct->enable_category_substring_match);
 #endif
 
@@ -2575,7 +2575,7 @@ void ndpi_exit_detection_module(struct ndpi_detection_module_struct *ndpi_struct
 
     if(ndpi_struct->protocols_ptree)
       ndpi_Destroy_Patricia((patricia_tree_t*)ndpi_struct->protocols_ptree, free_ptree_data);
-    
+
     if(ndpi_struct->udpRoot != NULL)
       ndpi_tdestroy(ndpi_struct->udpRoot, ndpi_free);
     if(ndpi_struct->tcpRoot != NULL)
@@ -2621,7 +2621,7 @@ void ndpi_exit_detection_module(struct ndpi_detection_module_struct *ndpi_struct
 
     if(ndpi_struct->custom_categories.hostnames_hash)
       ht_free((hashtable_t*)ndpi_struct->custom_categories.hostnames_hash);
-        
+
     ndpi_free(ndpi_struct);
   }
 }
@@ -2634,7 +2634,7 @@ int ndpi_get_protocol_id_master_proto(struct ndpi_detection_module_struct *ndpi_
 				      u_int16_t** udp_master_proto) {
   if(protocol_id >= (NDPI_MAX_SUPPORTED_PROTOCOLS+NDPI_MAX_NUM_CUSTOM_PROTOCOLS)) {
     *tcp_master_proto = ndpi_struct->proto_defaults[NDPI_PROTOCOL_UNKNOWN].master_tcp_protoId,
-      *udp_master_proto = ndpi_struct->proto_defaults[NDPI_PROTOCOL_UNKNOWN].master_udp_protoId;    
+      *udp_master_proto = ndpi_struct->proto_defaults[NDPI_PROTOCOL_UNKNOWN].master_udp_protoId;
     return(-1);
   }
 
@@ -2782,7 +2782,7 @@ char * strsep(char **sp, char *sep)
 
 /* ******************************************************************** */
 
-int ndpi_handle_rule(struct ndpi_detection_module_struct *ndpi_mod, 
+int ndpi_handle_rule(struct ndpi_detection_module_struct *ndpi_mod,
 		     char* rule, u_int8_t do_add) {
   char *at, *proto, *elem;
   ndpi_proto_defaults_t *def;
@@ -2917,7 +2917,7 @@ int ndpi_load_protocols_file(struct ndpi_detection_module_struct *ndpi_mod, char
 
   if(buffer == NULL) {
     NDPI_LOG_ERR(ndpi_mod, "Memory allocation failure");
-    goto close_fd;    
+    goto close_fd;
   }
 
   while(fd) {
@@ -3820,18 +3820,18 @@ static int ndpi_init_packet_header(struct ndpi_detection_module_struct *ndpi_str
 	 && flow->detected_protocol_stack[0] == NDPI_PROTOCOL_UNKNOWN) {
 	u_int8_t backup;
 	u_int16_t backup1, backup2;
-	
+
 	if(flow->http.url)          ndpi_free(flow->http.url);
 	if(flow->http.content_type) ndpi_free(flow->http.content_type);
 
 	backup  = flow->num_processed_pkts;
 	backup1 = flow->guessed_protocol_id;
-	backup2 = flow->guessed_host_protocol_id;	
+	backup2 = flow->guessed_host_protocol_id;
 	memset(flow, 0, sizeof(*(flow)));
 	flow->num_processed_pkts = backup;
 	flow->guessed_protocol_id      = backup1;
 	flow->guessed_host_protocol_id = backup2;
-	
+
 	NDPI_LOG_DBG(ndpi_struct,
 		 "tcp syn packet for unknown protocol, reset detection state\n");
       }
@@ -4165,6 +4165,52 @@ static u_int16_t ndpi_guess_host_protocol_id(struct ndpi_detection_module_struct
 
 /* ********************************************************************************* */
 
+static ndpi_protocol ndpi_process_partial_detection(struct ndpi_detection_module_struct *ndpi_struct,
+						    struct ndpi_flow_struct *flow) {
+  ndpi_protocol ret;
+  ndpi_protocol_match_result ret_match;
+  
+  ret.master_protocol = flow->guessed_protocol_id;
+  ret.app_protocol    = ndpi_match_host_subprotocol(ndpi_struct, flow,
+						    (char *)flow->host_server_name,
+						    strlen((const char*)flow->host_server_name),
+						    &ret_match,
+						    flow->guessed_protocol_id);
+  ret.category = ret_match.protocol_category;
+
+  if(ret.app_protocol == NDPI_PROTOCOL_UNKNOWN)
+    ret.app_protocol = ret.master_protocol;
+  
+  ndpi_int_change_protocol(ndpi_struct, flow, ret.app_protocol, ret.master_protocol);
+  return(ret);
+}
+
+/* ********************************************************************************* */
+
+/*
+  You can call this function at any time in case of unknown match to see if there is
+  a partial match that has been prevented by the current nDPI preferences configuration
+ */
+ndpi_protocol ndpi_get_partial_detection(struct ndpi_detection_module_struct *ndpi_struct,
+					 struct ndpi_flow_struct *flow) {
+  if((flow->guessed_protocol_id == NDPI_PROTOCOL_HTTP)
+     && (ndpi_struct->http_dont_dissect_response == 0)
+     && (flow->host_server_name[0] != '\0')
+     && (!NDPI_ISSET(&flow->excluded_protocol_bitmask, flow->guessed_host_protocol_id)))
+    return(ndpi_process_partial_detection(ndpi_struct, flow));
+  else if((flow->guessed_protocol_id == NDPI_PROTOCOL_DNS)
+	  && (ndpi_struct->dns_dont_dissect_response == 0)
+	  && (flow->host_server_name[0] != '\0')
+	  && (!NDPI_ISSET(&flow->excluded_protocol_bitmask, flow->guessed_host_protocol_id)))
+    return(ndpi_process_partial_detection(ndpi_struct, flow));
+  else {
+    ndpi_protocol ret = { NDPI_PROTOCOL_UNKNOWN, NDPI_PROTOCOL_UNKNOWN, NDPI_PROTOCOL_CATEGORY_UNSPECIFIED };
+    return(ret);
+  }
+}
+
+/* ********************************************************************************* */
+
 ndpi_protocol ndpi_detection_giveup(struct ndpi_detection_module_struct *ndpi_struct,
 				    struct ndpi_flow_struct *flow, u_int8_t enable_guess) {
   ndpi_protocol ret = { NDPI_PROTOCOL_UNKNOWN, NDPI_PROTOCOL_UNKNOWN, NDPI_PROTOCOL_CATEGORY_UNSPECIFIED };
@@ -4177,12 +4223,19 @@ ndpi_protocol ndpi_detection_giveup(struct ndpi_detection_module_struct *ndpi_st
 
     if(flow->guessed_protocol_id == NDPI_PROTOCOL_STUN)
       goto check_stun_export;
-    else if((flow->l4.tcp.ssl_seen_client_cert == 1) && (flow->protos.stun_ssl.ssl.client_certificate[0] != '\0')) {
+    else if((flow->l4.tcp.ssl_seen_client_cert == 1)
+	    && (flow->protos.stun_ssl.ssl.client_certificate[0] != '\0')) {
       ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_SSL, NDPI_PROTOCOL_UNKNOWN);
     } else {
-      if(!enable_guess)
-	return(ret);
-      
+      ndpi_protocol ret_g = ndpi_get_partial_detection(ndpi_struct, flow);
+
+      if(ret_g.master_protocol != NDPI_PROTOCOL_UNKNOWN)
+	return(ret_g);
+      else {
+	if(!enable_guess)
+	  return(ret);
+      }
+
       if((flow->guessed_protocol_id == NDPI_PROTOCOL_UNKNOWN)
 	 && (flow->packet.l4_protocol == IPPROTO_TCP)
 	 && (flow->l4.tcp.ssl_stage > 1))
@@ -4196,7 +4249,7 @@ ndpi_protocol ndpi_detection_giveup(struct ndpi_detection_module_struct *ndpi_st
 	     && is_udp_guessable_protocol(guessed_host_protocol_id)
 	     ))
 	flow->guessed_host_protocol_id = guessed_host_protocol_id = NDPI_PROTOCOL_UNKNOWN;
-      
+
       /* Ignore guessed protocol if they have been discarded */
       if((guessed_protocol_id != NDPI_PROTOCOL_UNKNOWN)
 	 // && (guessed_host_protocol_id == NDPI_PROTOCOL_UNKNOWN)
@@ -4207,7 +4260,6 @@ ndpi_protocol ndpi_detection_giveup(struct ndpi_detection_module_struct *ndpi_st
 
       if((guessed_protocol_id != NDPI_PROTOCOL_UNKNOWN)
 	 || (guessed_host_protocol_id != NDPI_PROTOCOL_UNKNOWN)) {
-
 	if((guessed_protocol_id == 0)
 	   && (flow->protos.stun_ssl.stun.num_binding_requests > 0)
 	   && (flow->protos.stun_ssl.stun.num_processed_pkts > 0))
@@ -4246,7 +4298,7 @@ ndpi_protocol ndpi_detection_giveup(struct ndpi_detection_module_struct *ndpi_st
     else if(ret.app_protocol == NDPI_PROTOCOL_GOOGLE)
       ret.app_protocol = NDPI_PROTOCOL_HANGOUT;
   }
-  
+
   if(enable_guess
      && (ret.app_protocol == NDPI_PROTOCOL_UNKNOWN)
      && flow->packet.iph /* Guess only IPv4 */
@@ -4260,7 +4312,7 @@ ndpi_protocol ndpi_detection_giveup(struct ndpi_detection_module_struct *ndpi_st
 					 ntohl(flow->packet.iph->daddr),
 					 ntohs(flow->packet.udp ? flow->packet.udp->dest : flow->packet.tcp->dest)
 					 );
-  
+
   ndpi_fill_protocol_category(ndpi_struct, flow, &ret);
 
   return(ret);
@@ -4353,15 +4405,15 @@ int ndpi_load_hostname_category(struct ndpi_detection_module_struct *ndpi_struct
 
     if(ndpi_struct->custom_categories.hostnames_hash)
       ht_set((hashtable_t*)ndpi_struct->custom_categories.hostnames_hash, name, (u_int16_t)category);
-    
-    return(0);    
+
+    return(0);
   } else {
     AC_PATTERN_t ac_pattern;
-    
+
     /* printf("===> Loading %s as %u\n", name, category); */
 
     memset(&ac_pattern, 0, sizeof(ac_pattern));
-    
+
 #ifdef HAVE_HYPERSCAN
     {
       struct hs_list *h = (struct hs_list*)malloc(sizeof(struct hs_list));
@@ -4401,7 +4453,7 @@ int ndpi_load_hostname_category(struct ndpi_detection_module_struct *ndpi_struct
     ac_automata_add(ndpi_struct->custom_categories.hostnames_shadow.ac_automa, &ac_pattern);
 #endif
   }
-  
+
   return(0);
 }
 
@@ -4481,16 +4533,16 @@ int ndpi_enable_loaded_categories(struct ndpi_detection_module_struct *ndpi_str)
     ndpi_str->custom_categories.hostnames_shadow.ac_automa = ac_automata_init(ac_match_handler);
 #endif
   }
-  
+
   if(ndpi_str->custom_categories.ipAddresses != NULL)
-    ndpi_Destroy_Patricia((patricia_tree_t*)ndpi_str->custom_categories.ipAddresses, 
+    ndpi_Destroy_Patricia((patricia_tree_t*)ndpi_str->custom_categories.ipAddresses,
 			  free_ptree_data);
 
   ndpi_str->custom_categories.ipAddresses = ndpi_str->custom_categories.ipAddresses_shadow;
   ndpi_str->custom_categories.ipAddresses_shadow = ndpi_New_Patricia(32 /* IPv4 */);
 
   ndpi_str->custom_categories.categories_loaded = 1;
-  
+
   return(0);
 }
 
@@ -4582,7 +4634,7 @@ ndpi_protocol ndpi_detection_process_packet(struct ndpi_detection_module_struct
     return(ret);
 
   flow->num_processed_pkts++;
-  
+
   if(flow->server_id == NULL) flow->server_id = dst; /* Default */
   if(flow->detected_protocol_stack[0] != NDPI_PROTOCOL_UNKNOWN)
     goto ret_protocols;
@@ -4705,7 +4757,7 @@ ndpi_protocol ndpi_detection_process_packet(struct ndpi_detection_module_struct
 	break;
       }
     }
-    
+
     ndpi_check_flow_func(ndpi_struct, flow, &ndpi_selection_packet);
     ndpi_fill_protocol_category(ndpi_struct, flow, &ret);
 
@@ -4969,22 +5021,22 @@ void ndpi_parse_packet_line_info(struct ndpi_detection_module_struct *ndpi_struc
 	packet->http_response.ptr = &packet->line[0].ptr[NDPI_STATICSTRING_LEN("HTTP/1.1 ")];
 	packet->http_response.len = packet->line[0].len - NDPI_STATICSTRING_LEN("HTTP/1.1 ");
 	packet->http_num_headers++;
-	
+
 	/* Set server HTTP response code */
 	if(packet->payload_packet_len >= 12) {
 	  char buf[4];
-	  
+
 	  /* Set server HTTP response code */
 	  strncpy(buf, (char*)&packet->payload[9], 3);
 	  buf[3] = '\0';
-	  
+
 	  flow->http.response_status_code = atoi(buf);
 	  /* https://en.wikipedia.org/wiki/List_of_HTTP_status_codes */
 	  if((flow->http.response_status_code < 100) || (flow->http.response_status_code > 509))
 	    flow->http.response_status_code = 0; /* Out of range */
 	}
       }
-      
+
       /* "Server:" header line in HTTP response */
       if(packet->line[packet->parsed_lines].len > NDPI_STATICSTRING_LEN("Server:") + 1
 	     && strncasecmp((const char *)packet->line[packet->parsed_lines].ptr, "Server:", NDPI_STATICSTRING_LEN("Server:")) == 0) {
@@ -5403,7 +5455,7 @@ void ndpi_int_change_protocol(struct ndpi_detection_module_struct *ndpi_struct,
       }
     }
   }
-  
+
   ndpi_int_change_flow_protocol(ndpi_struct, flow,
 				upper_detected_protocol, lower_detected_protocol);
   ndpi_int_change_packet_protocol(ndpi_struct, flow,
@@ -5435,7 +5487,7 @@ void ndpi_int_reset_protocol(struct ndpi_flow_struct *flow) {
     int a;
 
     for(a = 0; a < NDPI_PROTOCOL_SIZE; a++)
-      flow->detected_protocol_stack[a] = NDPI_PROTOCOL_UNKNOWN;    
+      flow->detected_protocol_stack[a] = NDPI_PROTOCOL_UNKNOWN;
   }
 }
 
@@ -5657,10 +5709,10 @@ ndpi_protocol ndpi_guess_undetected_protocol(struct ndpi_detection_module_struct
 	ret.app_protocol = rc,
 	  ret.master_protocol = ndpi_guess_protocol_id(ndpi_struct, NULL, proto, sport,
 						       dport, &user_defined_proto);
-	
+
 	if(ret.app_protocol == ret.master_protocol)
 	  ret.master_protocol = NDPI_PROTOCOL_UNKNOWN;
-	
+
 	ret.category = ndpi_get_proto_category(ndpi_struct, ret);
 	return(ret);
       }
@@ -5674,7 +5726,7 @@ ndpi_protocol ndpi_guess_undetected_protocol(struct ndpi_detection_module_struct
 	;
       else {
 	ret.app_protocol = rc;
-	
+
 	if(rc == NDPI_PROTOCOL_SSL)
 	  goto check_guessed_skype;
 	else {
@@ -5697,7 +5749,7 @@ ndpi_protocol ndpi_guess_undetected_protocol(struct ndpi_detection_module_struct
     ret.app_protocol = ndpi_guess_protocol_id(ndpi_struct, NULL, proto, sport,
 					      dport, &user_defined_proto);
 
-  ret.category = ndpi_get_proto_category(ndpi_struct, ret);  
+  ret.category = ndpi_get_proto_category(ndpi_struct, ret);
   return(ret);
 }
 
@@ -5939,7 +5991,7 @@ char* ndpi_strnstr(const char *s, const char *find, size_t slen) {
     } while(strncmp(s, find, len) != 0);
     s--;
   }
-  
+
   return((char *)s);
 }
 
@@ -5987,7 +6039,7 @@ int ndpi_match_string_subprotocol(struct ndpi_detection_module_struct *ndpi_stru
   ndpi_automa *automa = is_host_match ? &ndpi_struct->host_automa :
     &ndpi_struct->content_automa;
   AC_REP_t match = { NDPI_PROTOCOL_UNKNOWN, NDPI_PROTOCOL_CATEGORY_UNSPECIFIED, NDPI_PROTOCOL_UNRATED };
-  
+
   if((automa->ac_automa == NULL) || (string_to_match_len == 0))
     return(NDPI_PROTOCOL_UNKNOWN);
 
@@ -5995,15 +6047,15 @@ int ndpi_match_string_subprotocol(struct ndpi_detection_module_struct *ndpi_stru
     ac_automata_finalize((AC_AUTOMATA_t*)automa->ac_automa);
     automa->ac_automa_finalized = 1;
   }
-  
+
   ac_input_text.astring = string_to_match, ac_input_text.length = string_to_match_len;
   ac_automata_search(((AC_AUTOMATA_t*)automa->ac_automa), &ac_input_text, &match);
   ac_automata_reset(((AC_AUTOMATA_t*)automa->ac_automa));
-  
+
   ret_match->protocol_id = match.number,
     ret_match->protocol_category = match.category,
     ret_match->protocol_breed = match.breed;
-    
+
   return(match.number);
 }
 
@@ -6252,9 +6304,9 @@ struct ndpi_lru_cache* ndpi_lru_cache_init(u_int32_t num_entries) {
   if(!c->entries) {
     free(c);
     return(NULL);
-  } else  
+  } else
     c->num_entries = num_entries;
-  
+
   return(c);
 }
 
diff --git a/src/lib/protocols/dns.c b/src/lib/protocols/dns.c
index b99e5a5da..1c2593feb 100644
--- a/src/lib/protocols/dns.c
+++ b/src/lib/protocols/dns.c
@@ -153,8 +153,10 @@ void ndpi_search_dns(struct ndpi_detection_module_struct *ndpi_struct, struct nd
 	       || ((dns_header.authority_rrs > 0) && (dns_header.authority_rrs <= NDPI_MAX_DNS_REQUESTS))
 	       || ((dns_header.additional_rrs > 0) && (dns_header.additional_rrs <= NDPI_MAX_DNS_REQUESTS)))
 	   ) {
-	  /* This is a good reply */
-	  if(ndpi_struct->dns_dont_dissect_response == 0) {
+	  /* This is a good reply: we dissect it both for request and response */
+
+	  /* Leave the statement below commented necessary in case of call to ndpi_get_partial_detection() */
+	  /* if(ndpi_struct->dns_dont_dissect_response == 0) */ {
 	    x++;
 
 	    if(flow->packet.payload[x] != '\0') {
diff --git a/src/lib/protocols/http.c b/src/lib/protocols/http.c
index fc392c2b7..33ef9e2ed 100644
--- a/src/lib/protocols/http.c
+++ b/src/lib/protocols/http.c
@@ -157,7 +157,9 @@ static void check_content_type_and_change_protocol(struct ndpi_detection_module_
   }
 #endif
 
-  if(!ndpi_struct->http_dont_dissect_response) {
+  /* Leave the statement below commented necessary in case of call to ndpi_get_partial_detection() */
+  
+  /* if(!ndpi_struct->http_dont_dissect_response) */ {
     if((flow->http.url == NULL)
        && (packet->http_url_name.len > 0)
        && (packet->host_line.len > 0)) {
-- 
cgit v1.2.3


From 6693151052a98e6eddc722c139886d7fe84f35e4 Mon Sep 17 00:00:00 2001
From: Luca Deri <deri@ntop.org>
Date: Mon, 1 Apr 2019 16:12:44 +0200
Subject: Added custom category support to ndpi_get_partial_detection()

---
 example/ndpiReader.c | 3 ++-
 src/lib/ndpi_main.c  | 8 ++++----
 2 files changed, 6 insertions(+), 5 deletions(-)

(limited to 'src/lib/ndpi_main.c')

diff --git a/example/ndpiReader.c b/example/ndpiReader.c
index 72eaecca4..0b0fa889b 100644
--- a/example/ndpiReader.c
+++ b/example/ndpiReader.c
@@ -1505,7 +1505,8 @@ static void setupDetection(u_int16_t thread_id, pcap_t * pcap_handle) {
 	  if(category) {
 	    int fields[4];
 
-	    // printf("Loading %s\t%s\n", name, category);
+
+	    if(verbose) printf("[Category] Loading %s\t%s\n", name, category);
 
 	    if(sscanf(name, "%d.%d.%d.%d", &fields[0], &fields[1], &fields[2], &fields[3]) == 4)
 	      ndpi_load_ip_category(ndpi_thread_info[thread_id].workflow->ndpi_struct,
diff --git a/src/lib/ndpi_main.c b/src/lib/ndpi_main.c
index acbe5b8ec..cc5637cbc 100644
--- a/src/lib/ndpi_main.c
+++ b/src/lib/ndpi_main.c
@@ -2474,7 +2474,7 @@ static int hyperscanCustomEventHandler(unsigned int id,
 /* *********************************************** */
 
 int ndpi_match_custom_category(struct ndpi_detection_module_struct *ndpi_struct,
-				      char *name, unsigned long *id) {
+			       char *name, unsigned long *id) {
 #ifdef DEBUG
   printf("[NDPI] %s(%s) [enable_category_substring_match: %u]\n",
 	 __FUNCTION__, name, ndpi_struct->enable_category_substring_match);
@@ -2518,7 +2518,7 @@ int ndpi_match_custom_category(struct ndpi_detection_module_struct *ndpi_struct,
 /* *********************************************** */
 
 int ndpi_get_custom_category_match(struct ndpi_detection_module_struct *ndpi_struct,
-				      char *name_or_ip, unsigned long *id) {
+				   char *name_or_ip, unsigned long *id) {
   char ipbuf[64];
   struct in_addr pin;
 
@@ -4180,7 +4180,8 @@ static ndpi_protocol ndpi_process_partial_detection(struct ndpi_detection_module
 
   if(ret.app_protocol == NDPI_PROTOCOL_UNKNOWN)
     ret.app_protocol = ret.master_protocol;
-  
+
+  ndpi_fill_protocol_category(ndpi_struct, flow, &ret);
   ndpi_int_change_protocol(ndpi_struct, flow, ret.app_protocol, ret.master_protocol);
   return(ret);
 }
@@ -6090,7 +6091,6 @@ static int ndpi_automa_match_string_subprotocol(struct ndpi_detection_module_str
   matching_protocol_id = ndpi_match_string_subprotocol(ndpi_struct, string_to_match,
 						       string_to_match_len, ret_match,
 						       is_host_match);
-
 #else
   struct hs *hs = (struct hs*)ndpi_struct->hyperscan;
   hs_error_t status;
-- 
cgit v1.2.3


From 153c77c2cd28d52d6b459263dea3ce988ceccd3c Mon Sep 17 00:00:00 2001
From: Luca Deri <deri@ntop.org>
Date: Mon, 1 Apr 2019 18:40:14 +0200
Subject: Improvements with category detection

---
 src/lib/ndpi_main.c | 26 +++++++++++++++++++++-----
 1 file changed, 21 insertions(+), 5 deletions(-)

(limited to 'src/lib/ndpi_main.c')

diff --git a/src/lib/ndpi_main.c b/src/lib/ndpi_main.c
index cc5637cbc..866f65a10 100644
--- a/src/lib/ndpi_main.c
+++ b/src/lib/ndpi_main.c
@@ -4176,13 +4176,18 @@ static ndpi_protocol ndpi_process_partial_detection(struct ndpi_detection_module
 						    strlen((const char*)flow->host_server_name),
 						    &ret_match,
 						    flow->guessed_protocol_id);
-  ret.category = ret_match.protocol_category;
 
+  if(flow->category != NDPI_PROTOCOL_CATEGORY_UNSPECIFIED)
+    ret.category = flow->category;
+  else
+    ret.category = ret_match.protocol_category;
+  
   if(ret.app_protocol == NDPI_PROTOCOL_UNKNOWN)
     ret.app_protocol = ret.master_protocol;
 
   ndpi_fill_protocol_category(ndpi_struct, flow, &ret);
   ndpi_int_change_protocol(ndpi_struct, flow, ret.app_protocol, ret.master_protocol);
+
   return(ret);
 }
 
@@ -4205,7 +4210,12 @@ ndpi_protocol ndpi_get_partial_detection(struct ndpi_detection_module_struct *nd
 	  && (!NDPI_ISSET(&flow->excluded_protocol_bitmask, flow->guessed_host_protocol_id)))
     return(ndpi_process_partial_detection(ndpi_struct, flow));
   else {
-    ndpi_protocol ret = { NDPI_PROTOCOL_UNKNOWN, NDPI_PROTOCOL_UNKNOWN, NDPI_PROTOCOL_CATEGORY_UNSPECIFIED };
+    ndpi_protocol ret = { NDPI_PROTOCOL_UNKNOWN,
+			  NDPI_PROTOCOL_UNKNOWN,
+			  NDPI_PROTOCOL_CATEGORY_UNSPECIFIED };
+
+    if(flow) ret.category = flow->category;
+    
     return(ret);
   }
 }
@@ -4216,8 +4226,11 @@ ndpi_protocol ndpi_detection_giveup(struct ndpi_detection_module_struct *ndpi_st
 				    struct ndpi_flow_struct *flow, u_int8_t enable_guess) {
   ndpi_protocol ret = { NDPI_PROTOCOL_UNKNOWN, NDPI_PROTOCOL_UNKNOWN, NDPI_PROTOCOL_CATEGORY_UNSPECIFIED };
 
-  if(flow == NULL) return(ret);
-
+  if(flow == NULL)
+    return(ret);
+  else
+    ret.category = flow->category;
+  
   /* TODO: add the remaining stage_XXXX protocols */
   if(flow->detected_protocol_stack[0] == NDPI_PROTOCOL_UNKNOWN) {
     u_int16_t guessed_protocol_id, guessed_host_protocol_id;
@@ -4631,9 +4644,12 @@ ndpi_protocol ndpi_detection_process_packet(struct ndpi_detection_module_struct
   if(ndpi_struct->ndpi_log_level >= NDPI_LOG_TRACE)
   	NDPI_LOG(flow ? flow->detected_protocol_stack[0]:NDPI_PROTOCOL_UNKNOWN,
 		 ndpi_struct, NDPI_LOG_TRACE, "START packet processing\n");
+
   if(flow == NULL)
     return(ret);
-
+  else
+    ret.category = flow->category;
+      
   flow->num_processed_pkts++;
 
   if(flow->server_id == NULL) flow->server_id = dst; /* Default */
-- 
cgit v1.2.3


From a455e0d8d48a479d5d43023316cb97cb012b78fa Mon Sep 17 00:00:00 2001
From: Luca Deri <deri@ntop.org>
Date: Tue, 2 Apr 2019 23:01:03 +0200
Subject: Improved flow category guess

---
 src/include/ndpi_typedefs.h |   2 +-
 src/lib/ndpi_main.c         | 415 ++++++++++++++++++++++----------------------
 2 files changed, 212 insertions(+), 205 deletions(-)

(limited to 'src/lib/ndpi_main.c')

diff --git a/src/include/ndpi_typedefs.h b/src/include/ndpi_typedefs.h
index 1bd8fd2db..fc80a675b 100644
--- a/src/include/ndpi_typedefs.h
+++ b/src/include/ndpi_typedefs.h
@@ -1027,7 +1027,7 @@ struct ndpi_flow_struct {
   u_int16_t protocol_stack_info;
 
   /* init parameter, internal used to set up timestamp,... */
-  u_int16_t guessed_protocol_id, guessed_host_protocol_id, guessed_category;
+  u_int16_t guessed_protocol_id, guessed_host_protocol_id, guessed_category, guessed_header_category;
   u_int8_t protocol_id_already_guessed:1, host_already_guessed:1, init_finished:1, setup_packet_direction:1, packet_direction:1, check_extra_packets:1;
 
   /*
diff --git a/src/lib/ndpi_main.c b/src/lib/ndpi_main.c
index 866f65a10..bdd28f666 100644
--- a/src/lib/ndpi_main.c
+++ b/src/lib/ndpi_main.c
@@ -320,110 +320,110 @@ int strncasecmp(s1, s2, n)
 
 /* Keep it in order and in sync with ndpi_protocol_category_t in ndpi_typedefs.h */
 static const char* categories[] = {
-   "Unspecified",
-   "Media",
-   "VPN",
-   "Email",
-   "DataTransfer",
-   "Web",
-   "SocialNetwork",
-   "Download-FileTransfer-FileSharing",
-   "Game",
-   "Chat",
-   "VoIP",
-   "Database",
-   "RemoteAccess",
-   "Cloud",
-   "Network",
-   "Collaborative",
-   "RPC",
-   "Streaming",
-   "System",
-   "SoftwareUpdate",
-   "",
-   "",
-   "",
-   "",
-   "",
-   "Music",
-   "Video",
-   "Shopping",
-   "Productivity",
-   "FileSharing",
-   "",
-   "",
-   "",
-   "",
-   "",
-   "",
-   "",
-   "",
-   "",
-   "",
-   "",
-   "",
-   "",
-   "",
-   "",
-   "",
-   "",
-   "",
-   "",
-   "",
-   "",
-   "",
-   "",
-   "",
-   "",
-   "",
-   "",
-   "",
-   "",
-   "",
-   "",
-   "",
-   "",
-   "",
-   "",
-   "",
-   "",
-   "",
-   "",
-   "",
-   "",
-   "",
-   "",
-   "",
-   "",
-   "",
-   "",
-   "",
-   "",
-   "",
-   "",
-   "",
-   "",
-   "",
-   "",
-   "",
-   "",
-   "",
-   "",
-   "",
-   "",
-   "",
-   "",
-   "",
-   "",
-   "",
-   "",
-   "",
-   "",
-   "Mining", /* 99 */
-   "Malware",
-   "Advertisement",
-   "Banned_Site",
-   "Site_Unavailable"
+  "Unspecified",
+  "Media",
+  "VPN",
+  "Email",
+  "DataTransfer",
+  "Web",
+  "SocialNetwork",
+  "Download-FileTransfer-FileSharing",
+  "Game",
+  "Chat",
+  "VoIP",
+  "Database",
+  "RemoteAccess",
+  "Cloud",
+  "Network",
+  "Collaborative",
+  "RPC",
+  "Streaming",
+  "System",
+  "SoftwareUpdate",
+  "",
+  "",
+  "",
+  "",
+  "",
+  "Music",
+  "Video",
+  "Shopping",
+  "Productivity",
+  "FileSharing",
+  "",
+  "",
+  "",
+  "",
+  "",
+  "",
+  "",
+  "",
+  "",
+  "",
+  "",
+  "",
+  "",
+  "",
+  "",
+  "",
+  "",
+  "",
+  "",
+  "",
+  "",
+  "",
+  "",
+  "",
+  "",
+  "",
+  "",
+  "",
+  "",
+  "",
+  "",
+  "",
+  "",
+  "",
+  "",
+  "",
+  "",
+  "",
+  "",
+  "",
+  "",
+  "",
+  "",
+  "",
+  "",
+  "",
+  "",
+  "",
+  "",
+  "",
+  "",
+  "",
+  "",
+  "",
+  "",
+  "",
+  "",
+  "",
+  "",
+  "",
+  "",
+  "",
+  "",
+  "",
+  "",
+  "",
+  "",
+  "",
+  "",
+  "Mining", /* 99 */
+  "Malware",
+  "Advertisement",
+  "Banned_Site",
+  "Site_Unavailable"
 };
 
 /* ****************************************** */
@@ -597,11 +597,11 @@ void ndpi_set_proto_category(struct ndpi_detection_module_struct *ndpi_mod,
 
   Example:
   - DNS is informative as if we see a DNS request for www.facebook.com, the
-    returned protocol is DNS.Facebook, but Facebook isn't a real subprotocol but
-    rather it indicates a query for Facebook and not Facebook traffic.
+  returned protocol is DNS.Facebook, but Facebook isn't a real subprotocol but
+  rather it indicates a query for Facebook and not Facebook traffic.
   - HTTP/SSL are NOT informative as SSL.Facebook (likely) means that this is
-    SSL (HTTPS) traffic containg Facebook traffic.
- */
+  SSL (HTTPS) traffic containg Facebook traffic.
+*/
 u_int8_t ndpi_is_subprotocol_informative(struct ndpi_detection_module_struct *ndpi_mod,
 					 u_int16_t protoId) {
   if(protoId >= NDPI_MAX_SUPPORTED_PROTOCOLS+NDPI_MAX_NUM_CUSTOM_PROTOCOLS)
@@ -619,21 +619,21 @@ u_int8_t ndpi_is_subprotocol_informative(struct ndpi_detection_module_struct *nd
 /* ********************************************************************************** */
 
 void ndpi_exclude_protocol(struct ndpi_detection_module_struct *ndpi_struct,
-                                  struct ndpi_flow_struct *flow,
-                                  u_int16_t protocol_id,
-                                  const char *_file, const char *_func,int _line) {
+			   struct ndpi_flow_struct *flow,
+			   u_int16_t protocol_id,
+			   const char *_file, const char *_func,int _line) {
 
   if(protocol_id < NDPI_MAX_SUPPORTED_PROTOCOLS+NDPI_MAX_NUM_CUSTOM_PROTOCOLS) {
 #ifdef NDPI_ENABLE_DEBUG_MESSAGES
-	if( ndpi_struct &&
-	     ndpi_struct->ndpi_log_level >= NDPI_LOG_DEBUG &&
-    	     ndpi_struct->ndpi_debug_printf != NULL) {
+    if( ndpi_struct &&
+	ndpi_struct->ndpi_log_level >= NDPI_LOG_DEBUG &&
+	ndpi_struct->ndpi_debug_printf != NULL) {
 
-		(*(ndpi_struct->ndpi_debug_printf))(protocol_id, ndpi_struct, NDPI_LOG_DEBUG,
-		 _file, _func, _line, "exclude %s\n",ndpi_get_proto_name(ndpi_struct, protocol_id));
-	}
+      (*(ndpi_struct->ndpi_debug_printf))(protocol_id, ndpi_struct, NDPI_LOG_DEBUG,
+					  _file, _func, _line, "exclude %s\n",ndpi_get_proto_name(ndpi_struct, protocol_id));
+    }
 #endif
-	NDPI_ADD_PROTOCOL_TO_BITMASK(flow->excluded_protocol_bitmask, protocol_id);
+    NDPI_ADD_PROTOCOL_TO_BITMASK(flow->excluded_protocol_bitmask, protocol_id);
   }
 }
 
@@ -736,7 +736,7 @@ static void addDefaultPort(struct ndpi_detection_module_struct *ndpi_mod,
 
     if(ret != node) {
       NDPI_LOG_DBG(ndpi_mod, "[NDPI] %s:%d found duplicate for port %u: overwriting it with new value\n",
-		      _func, _line, port);
+		   _func, _line, port);
 
       ret->proto = def;
       ndpi_free(node);
@@ -860,8 +860,8 @@ void ndpi_init_protocol_match(struct ndpi_detection_module_struct *ndpi_mod,
   if(ndpi_mod->proto_defaults[match->protocol_id].protoName == NULL) {
     if(match->protocol_id == NDPI_PROTOCOL_GENERIC)
       ndpi_mod->proto_defaults[match->protocol_id].protoName   = ndpi_strdup(NDPI_CONST_GENERIC_PROTOCOL_NAME);
-   else
-     ndpi_mod->proto_defaults[match->protocol_id].protoName    = ndpi_strdup(match->proto_name);
+    else
+      ndpi_mod->proto_defaults[match->protocol_id].protoName    = ndpi_strdup(match->proto_name);
 
     ndpi_mod->proto_defaults[match->protocol_id].protoId       = match->protocol_id;
     ndpi_mod->proto_defaults[match->protocol_id].protoCategory = match->protocol_category;
@@ -2113,7 +2113,7 @@ static int ac_match_handler(AC_MATCH_t *m, AC_TEXT_t *txt, AC_REP_t *match) {
       The patch below allows in case of pattern ws.amazon.com
       to avoid matching aws.amazon.com whereas a.ws.amazon.com
       has to match
-     */
+    */
     if(whatfound && (whatfound != buf)
        && (m->patterns->astring[0] != '.')  /* The searched patter does not start with . */
        && strchr(m->patterns->astring, '.') /* The matched pattern has a . (e.g. numeric or sym IPs) */
@@ -2133,8 +2133,8 @@ static int ac_match_handler(AC_MATCH_t *m, AC_TEXT_t *txt, AC_REP_t *match) {
      || (strncmp(buf, m->patterns->astring, min_len) == 0) /* begins with */
      ) {
 #ifdef MATCH_DEBUG
-      printf("Found match [%s][%s] [len: %u][proto_id: %u]\n",
-	     buf, m->patterns->astring, min_len , *matching_protocol_id);
+    printf("Found match [%s][%s] [len: %u][proto_id: %u]\n",
+	   buf, m->patterns->astring, min_len , *matching_protocol_id);
 #endif
     return(1); /* If the pattern found matches the string at the beginning we stop here */
   } else
@@ -2269,23 +2269,23 @@ void set_ndpi_free(void  (*__ndpi_free)(void *ptr))       { _ndpi_free = __ndpi_
 void set_ndpi_flow_free(void  (*__ndpi_flow_free)(void *ptr))       { _ndpi_flow_free = __ndpi_flow_free; }
 
 void ndpi_debug_printf(unsigned int proto, struct ndpi_detection_module_struct *ndpi_str,
-		ndpi_log_level_t log_level, const char *file_name, const char *func_name, int line_number,
-		const char * format, ...)
+		       ndpi_log_level_t log_level, const char *file_name, const char *func_name, int line_number,
+		       const char * format, ...)
 {
 #ifdef NDPI_ENABLE_DEBUG_MESSAGES
   va_list args;
 #define MAX_STR_LEN 250
   char str[MAX_STR_LEN];
   if(ndpi_str != NULL && log_level > NDPI_LOG_ERROR  &&
-	proto > 0 && proto < NDPI_MAX_SUPPORTED_PROTOCOLS &&
-	!NDPI_ISSET(&ndpi_str->debug_bitmask,proto)) return;
+     proto > 0 && proto < NDPI_MAX_SUPPORTED_PROTOCOLS &&
+     !NDPI_ISSET(&ndpi_str->debug_bitmask,proto)) return;
   va_start(args, format);
   vsnprintf(str,sizeof(str)-1, format, args);
   va_end(args);
 
   if(ndpi_str != NULL) {
     printf("%s:%s:%-3u - [%s]: %s",
-	    file_name, func_name, line_number, ndpi_get_proto_name(ndpi_str, proto), str);
+	   file_name, func_name, line_number, ndpi_get_proto_name(ndpi_str, proto), str);
   } else {
     printf("Proto: %u, %s", proto, str);
   }
@@ -2679,7 +2679,7 @@ static ndpi_default_ports_tree_node_t* ndpi_get_guessed_protocol_id(struct ndpi_
   These are UDP protocols that must fit a single packet
   and thus that if have NOT been detected they cannot be guessed
   as they have been excluded
- */
+*/
 u_int8_t is_udp_guessable_protocol(u_int16_t l7_guessed_proto) {
   switch(l7_guessed_proto) {
   case NDPI_PROTOCOL_QUIC:
@@ -2829,7 +2829,7 @@ int ndpi_handle_rule(struct ndpi_detection_module_struct *ndpi_mod,
 
       if(ndpi_mod->ndpi_num_custom_protocols >= (NDPI_MAX_NUM_CUSTOM_PROTOCOLS-1)) {
 	NDPI_LOG_ERR(ndpi_mod, "Too many protocols defined (%u): skipping protocol %s\n",
-	       ndpi_mod->ndpi_num_custom_protocols, proto);
+		     ndpi_mod->ndpi_num_custom_protocols, proto);
 	return(-2);
       }
 
@@ -2983,17 +2983,17 @@ void ndpi_set_bitmask_protocol_detection(char * label,
   if(NDPI_COMPARE_PROTOCOL_TO_BITMASK(*detection_bitmask, ndpi_protocol_id) != 0) {
 #ifdef DEBUG
     NDPI_LOG_DBG2(ndpi_struct
-	"[NDPI] ndpi_set_bitmask_protocol_detection: %s : [callback_buffer] idx= %u, [proto_defaults] protocol_id=%u\n",
-	label, idx, ndpi_protocol_id);
+		  "[NDPI] ndpi_set_bitmask_protocol_detection: %s : [callback_buffer] idx= %u, [proto_defaults] protocol_id=%u\n",
+		  label, idx, ndpi_protocol_id);
 #endif
 
     if(ndpi_struct->proto_defaults[ndpi_protocol_id].protoIdx != 0) {
       NDPI_LOG_DBG2(ndpi_struct,
-		"[NDPI] Internal error: protocol %s/%u has been already registered\n", label, ndpi_protocol_id);
+		    "[NDPI] Internal error: protocol %s/%u has been already registered\n", label, ndpi_protocol_id);
 #ifdef DEBUG
     } else {
       NDPI_LOG_DBG2(ndpi_struct,
-		"[NDPI] Adding %s with protocol id %d\n", label, ndpi_protocol_id);
+		    "[NDPI] Adding %s with protocol id %d\n", label, ndpi_protocol_id);
 #endif
     }
 
@@ -3501,7 +3501,7 @@ void ndpi_set_protocol_detection_bitmask2(struct ndpi_detection_module_struct *n
   ndpi_struct->callback_buffer_size = a;
 
   NDPI_LOG_DBG2(ndpi_struct,
-	   "callback_buffer_size is %u\n", ndpi_struct->callback_buffer_size);
+		"callback_buffer_size is %u\n", ndpi_struct->callback_buffer_size);
 
   /* now build the specific buffer for tcp, udp and non_tcp_udp */
   ndpi_struct->callback_buffer_size_tcp_payload = 0;
@@ -3512,8 +3512,8 @@ void ndpi_set_protocol_detection_bitmask2(struct ndpi_detection_module_struct *n
 	   NDPI_SELECTION_BITMASK_PROTOCOL_INT_TCP_OR_UDP |
 	   NDPI_SELECTION_BITMASK_PROTOCOL_COMPLETE_TRAFFIC)) != 0) {
       if(_ndpi_debug_callbacks) NDPI_LOG_DBG2(ndpi_struct,
-	       "callback_buffer_tcp_payload, adding buffer %u as entry %u\n", a,
-	       ndpi_struct->callback_buffer_size_tcp_payload);
+					      "callback_buffer_tcp_payload, adding buffer %u as entry %u\n", a,
+					      ndpi_struct->callback_buffer_size_tcp_payload);
 
       memcpy(&ndpi_struct->callback_buffer_tcp_payload[ndpi_struct->callback_buffer_size_tcp_payload],
 	     &ndpi_struct->callback_buffer[a], sizeof(struct ndpi_call_function_struct));
@@ -3522,7 +3522,7 @@ void ndpi_set_protocol_detection_bitmask2(struct ndpi_detection_module_struct *n
       if((ndpi_struct->
 	  callback_buffer[a].ndpi_selection_bitmask & NDPI_SELECTION_BITMASK_PROTOCOL_HAS_PAYLOAD) == 0) {
 	if(_ndpi_debug_callbacks) NDPI_LOG_DBG2(ndpi_struct,
-		 "\tcallback_buffer_tcp_no_payload, additional adding buffer %u to no_payload process\n", a);
+						"\tcallback_buffer_tcp_no_payload, additional adding buffer %u to no_payload process\n", a);
 
 	memcpy(&ndpi_struct->callback_buffer_tcp_no_payload
 	       [ndpi_struct->callback_buffer_size_tcp_no_payload], &ndpi_struct->callback_buffer[a],
@@ -3539,7 +3539,7 @@ void ndpi_set_protocol_detection_bitmask2(struct ndpi_detection_module_struct *n
 								  NDPI_SELECTION_BITMASK_PROTOCOL_COMPLETE_TRAFFIC))
        != 0) {
       if(_ndpi_debug_callbacks) NDPI_LOG_DBG2(ndpi_struct,
-	       "callback_buffer_size_udp: adding buffer : %u as entry %u\n", a, ndpi_struct->callback_buffer_size_udp);
+					      "callback_buffer_size_udp: adding buffer : %u as entry %u\n", a, ndpi_struct->callback_buffer_size_udp);
 
       memcpy(&ndpi_struct->callback_buffer_udp[ndpi_struct->callback_buffer_size_udp],
 	     &ndpi_struct->callback_buffer[a], sizeof(struct ndpi_call_function_struct));
@@ -3556,7 +3556,7 @@ void ndpi_set_protocol_detection_bitmask2(struct ndpi_detection_module_struct *n
        || (ndpi_struct->
 	   callback_buffer[a].ndpi_selection_bitmask & NDPI_SELECTION_BITMASK_PROTOCOL_COMPLETE_TRAFFIC) != 0) {
       if(_ndpi_debug_callbacks) NDPI_LOG_DBG2(ndpi_struct,
-	       "callback_buffer_non_tcp_udp: adding buffer : %u as entry %u\n", a, ndpi_struct->callback_buffer_size_non_tcp_udp);
+					      "callback_buffer_non_tcp_udp: adding buffer : %u as entry %u\n", a, ndpi_struct->callback_buffer_size_non_tcp_udp);
 
       memcpy(&ndpi_struct->callback_buffer_non_tcp_udp[ndpi_struct->callback_buffer_size_non_tcp_udp],
 	     &ndpi_struct->callback_buffer[a], sizeof(struct ndpi_call_function_struct));
@@ -3744,11 +3744,10 @@ static int ndpi_init_packet_header(struct ndpi_detection_module_struct *ndpi_str
 #endif							/* NDPI_DETECTION_SUPPORT_IPV6 */
   }
 
-  if(flow) {
+  if(flow)
     ndpi_apply_flow_protocol_to_packet(flow, &flow->packet);
-  } else {
+  else
     ndpi_int_reset_packet_protocol(&flow->packet);
-  }
 
   l3len = flow->packet.l3_packet_len;
 
@@ -3756,7 +3755,7 @@ static int ndpi_init_packet_header(struct ndpi_detection_module_struct *ndpi_str
   if(flow->packet.iph != NULL) {
 #endif							/* NDPI_DETECTION_SUPPORT_IPV6 */
 
-    decaps_iph =flow->packet.iph;
+    decaps_iph = flow->packet.iph;
 
 #ifdef NDPI_DETECTION_SUPPORT_IPV6
   }
@@ -3833,7 +3832,7 @@ static int ndpi_init_packet_header(struct ndpi_detection_module_struct *ndpi_str
 	flow->guessed_host_protocol_id = backup2;
 
 	NDPI_LOG_DBG(ndpi_struct,
-		 "tcp syn packet for unknown protocol, reset detection state\n");
+		     "tcp syn packet for unknown protocol, reset detection state\n");
       }
     } else {
       /* tcp header not complete */
@@ -4057,7 +4056,7 @@ void check_ndpi_udp_flow_func(struct ndpi_detection_module_struct *ndpi_struct,
 	break; /* Stop after detecting the first protocol */
     } else
       if(_ndpi_debug_callbacks) NDPI_LOG_DBG2(ndpi_struct,
-	       "[UDP,SKIP] dissector of protocol as callback_buffer idx =  %d\n",a);
+					      "[UDP,SKIP] dissector of protocol as callback_buffer idx =  %d\n",a);
   }
 }
 
@@ -4169,7 +4168,7 @@ static ndpi_protocol ndpi_process_partial_detection(struct ndpi_detection_module
 						    struct ndpi_flow_struct *flow) {
   ndpi_protocol ret;
   ndpi_protocol_match_result ret_match;
-  
+
   ret.master_protocol = flow->guessed_protocol_id;
   ret.app_protocol    = ndpi_match_host_subprotocol(ndpi_struct, flow,
 						    (char *)flow->host_server_name,
@@ -4181,11 +4180,12 @@ static ndpi_protocol ndpi_process_partial_detection(struct ndpi_detection_module
     ret.category = flow->category;
   else
     ret.category = ret_match.protocol_category;
-  
+
   if(ret.app_protocol == NDPI_PROTOCOL_UNKNOWN)
     ret.app_protocol = ret.master_protocol;
 
   ndpi_fill_protocol_category(ndpi_struct, flow, &ret);
+  
   ndpi_int_change_protocol(ndpi_struct, flow, ret.app_protocol, ret.master_protocol);
 
   return(ret);
@@ -4196,7 +4196,7 @@ static ndpi_protocol ndpi_process_partial_detection(struct ndpi_detection_module
 /*
   You can call this function at any time in case of unknown match to see if there is
   a partial match that has been prevented by the current nDPI preferences configuration
- */
+*/
 ndpi_protocol ndpi_get_partial_detection(struct ndpi_detection_module_struct *ndpi_struct,
 					 struct ndpi_flow_struct *flow) {
   if((flow->guessed_protocol_id == NDPI_PROTOCOL_HTTP)
@@ -4215,7 +4215,7 @@ ndpi_protocol ndpi_get_partial_detection(struct ndpi_detection_module_struct *nd
 			  NDPI_PROTOCOL_CATEGORY_UNSPECIFIED };
 
     if(flow) ret.category = flow->category;
-    
+
     return(ret);
   }
 }
@@ -4230,7 +4230,7 @@ ndpi_protocol ndpi_detection_giveup(struct ndpi_detection_module_struct *ndpi_st
     return(ret);
   else
     ret.category = flow->category;
-  
+
   /* TODO: add the remaining stage_XXXX protocols */
   if(flow->detected_protocol_stack[0] == NDPI_PROTOCOL_UNKNOWN) {
     u_int16_t guessed_protocol_id, guessed_host_protocol_id;
@@ -4409,7 +4409,7 @@ void ndpi_load_ip_category(struct ndpi_detection_module_struct *ndpi_struct,
  *
  */
 int ndpi_load_hostname_category(struct ndpi_detection_module_struct *ndpi_struct,
-				 char *name, ndpi_protocol_category_t category) {
+				char *name, ndpi_protocol_category_t category) {
   if(name == NULL)
     return(-1);
 
@@ -4563,31 +4563,32 @@ int ndpi_enable_loaded_categories(struct ndpi_detection_module_struct *ndpi_str)
 /* ********************************************************************************* */
 
 int ndpi_fill_ip_protocol_category(struct ndpi_detection_module_struct *ndpi_struct,
-				 u_int32_t saddr,
-				 u_int32_t daddr,
-				 ndpi_protocol *ret) {
+				   u_int32_t saddr,
+				   u_int32_t daddr,
+				   ndpi_protocol *ret) {
   if(ndpi_struct->custom_categories.categories_loaded) {
-      prefix_t prefix;
-      patricia_node_t *node;
+    prefix_t prefix;
+    patricia_node_t *node;
+
+    /* Make sure all in network byte order otherwise compares wont work */
+    fill_prefix_v4(&prefix, (struct in_addr *)&saddr,
+		   32, ((patricia_tree_t*)ndpi_struct->protocols_ptree)->maxbits);
+    node = ndpi_patricia_search_best(ndpi_struct->custom_categories.ipAddresses, &prefix);
 
-      /* Make sure all in network byte order otherwise compares wont work */
-      fill_prefix_v4(&prefix, (struct in_addr *)&saddr,
+    if(!node) {
+      fill_prefix_v4(&prefix, (struct in_addr *)&daddr,
 		     32, ((patricia_tree_t*)ndpi_struct->protocols_ptree)->maxbits);
       node = ndpi_patricia_search_best(ndpi_struct->custom_categories.ipAddresses, &prefix);
+    }
 
-      if(!node) {
-	fill_prefix_v4(&prefix, (struct in_addr *)&daddr,
-		       32, ((patricia_tree_t*)ndpi_struct->protocols_ptree)->maxbits);
-	node = ndpi_patricia_search_best(ndpi_struct->custom_categories.ipAddresses, &prefix);
-      }
-
-      if(node) {
-	ret->category = (ndpi_protocol_category_t)node->value.user_value;
-	return 1;
-      }
+    if(node) {
+      ret->category = (ndpi_protocol_category_t)node->value.user_value;
+      return 1;
+    }
   }
 
   ret->category = ndpi_get_proto_category(ndpi_struct, *ret);
+
   return 0;
 }
 
@@ -4597,12 +4598,10 @@ void ndpi_fill_protocol_category(struct ndpi_detection_module_struct *ndpi_struc
 				 struct ndpi_flow_struct *flow,
 				 ndpi_protocol *ret) {
   if(ndpi_struct->custom_categories.categories_loaded) {
-    if(flow->packet.iph) {
-      if(ndpi_fill_ip_protocol_category(ndpi_struct, flow->packet.iph->saddr, flow->packet.iph->daddr, ret)) {
-        flow->category = ret->category;
-        return;
-      }
-    }
+    if(flow->guessed_header_category != NDPI_PROTOCOL_CATEGORY_UNSPECIFIED) {
+      flow->category = flow->guessed_header_category;
+      return;
+    }  
 
     if(flow->host_server_name[0] != '\0') {
       unsigned long id;
@@ -4642,14 +4641,14 @@ ndpi_protocol ndpi_detection_process_packet(struct ndpi_detection_module_struct
   ndpi_protocol ret = { NDPI_PROTOCOL_UNKNOWN, NDPI_PROTOCOL_UNKNOWN, NDPI_PROTOCOL_CATEGORY_UNSPECIFIED };
 
   if(ndpi_struct->ndpi_log_level >= NDPI_LOG_TRACE)
-  	NDPI_LOG(flow ? flow->detected_protocol_stack[0]:NDPI_PROTOCOL_UNKNOWN,
-		 ndpi_struct, NDPI_LOG_TRACE, "START packet processing\n");
+    NDPI_LOG(flow ? flow->detected_protocol_stack[0]:NDPI_PROTOCOL_UNKNOWN,
+	     ndpi_struct, NDPI_LOG_TRACE, "START packet processing\n");
 
   if(flow == NULL)
     return(ret);
   else
     ret.category = flow->category;
-      
+
   flow->num_processed_pkts++;
 
   if(flow->server_id == NULL) flow->server_id = dst; /* Default */
@@ -4731,6 +4730,14 @@ ndpi_protocol ndpi_detection_process_packet(struct ndpi_detection_module_struct
     flow->guessed_protocol_id = (int16_t) ndpi_guess_protocol_id(ndpi_struct, flow, protocol, sport, dport, &user_defined_proto);
     flow->guessed_host_protocol_id = ndpi_guess_host_protocol_id(ndpi_struct, flow);
 
+    if(ndpi_struct->custom_categories.categories_loaded && flow->packet.iph) {
+      ndpi_protocol ret;
+      
+      ndpi_fill_ip_protocol_category(ndpi_struct, flow->packet.iph->saddr, flow->packet.iph->daddr, &ret);
+      flow->guessed_header_category = ret.category;
+    } else
+      flow->guessed_header_category = NDPI_PROTOCOL_CATEGORY_UNSPECIFIED;
+    
     if(flow->guessed_protocol_id >= (NDPI_MAX_SUPPORTED_PROTOCOLS-1)) {
       /* This is a custom protocol and it has priority over everything else */
       ret.master_protocol = NDPI_PROTOCOL_UNKNOWN,
@@ -4812,7 +4819,7 @@ ndpi_protocol ndpi_detection_process_packet(struct ndpi_detection_module_struct
      && (ret.app_protocol == NDPI_PROTOCOL_UNKNOWN)
      && flow->packet.tcp
      && (flow->packet.tcp->syn == 0)
-    ) {
+     ) {
     /*
       This is a TCP flow
       - whose first packet is NOT a SYN
@@ -5025,7 +5032,7 @@ void ndpi_parse_packet_line_info(struct ndpi_detection_module_struct *ndpi_struc
 
   for(a = 0; a < packet->payload_packet_len; a++) {
     if((a + 1) == packet->payload_packet_len)
-	  return; /* Return if only one byte remains (prevent invalid reads past end-of-buffer) */
+      return; /* Return if only one byte remains (prevent invalid reads past end-of-buffer) */
 
     if(get_u_int16_t(packet->payload, a) == ntohs(0x0d0a)) { /* If end of line char sequence CR+NL "\r\n", process line */
       packet->line[packet->parsed_lines].len = (u_int16_t)(((unsigned long) &packet->payload[a]) - ((unsigned long) packet->line[packet->parsed_lines].ptr));
@@ -5056,8 +5063,8 @@ void ndpi_parse_packet_line_info(struct ndpi_detection_module_struct *ndpi_struc
 
       /* "Server:" header line in HTTP response */
       if(packet->line[packet->parsed_lines].len > NDPI_STATICSTRING_LEN("Server:") + 1
-	     && strncasecmp((const char *)packet->line[packet->parsed_lines].ptr, "Server:", NDPI_STATICSTRING_LEN("Server:")) == 0) {
-	    // some stupid clients omit a space and place the servername directly after the colon
+	 && strncasecmp((const char *)packet->line[packet->parsed_lines].ptr, "Server:", NDPI_STATICSTRING_LEN("Server:")) == 0) {
+	// some stupid clients omit a space and place the servername directly after the colon
         if(packet->line[packet->parsed_lines].ptr[NDPI_STATICSTRING_LEN("Server:")] == ' ') {
           packet->server_line.ptr =
             &packet->line[packet->parsed_lines].ptr[NDPI_STATICSTRING_LEN("Server:") + 1];
@@ -5072,7 +5079,7 @@ void ndpi_parse_packet_line_info(struct ndpi_detection_module_struct *ndpi_struc
       /* "Host:" header line in HTTP request */
       if(packet->line[packet->parsed_lines].len > 6
 	 && strncasecmp((const char *)packet->line[packet->parsed_lines].ptr,
-			     "Host:", 5) == 0) {
+			"Host:", 5) == 0) {
         // some stupid clients omit a space and place the hostname directly after the colon
         if(packet->line[packet->parsed_lines].ptr[5] == ' ') {
           packet->host_line.ptr = &packet->line[packet->parsed_lines].ptr[6];
@@ -5085,7 +5092,7 @@ void ndpi_parse_packet_line_info(struct ndpi_detection_module_struct *ndpi_struc
       }
       /* "X-Forwarded-For:" header line in HTTP request. Commonly used for HTTP proxies. */
       if(packet->line[packet->parsed_lines].len > 17
-	     && strncasecmp((const char *)packet->line[packet->parsed_lines].ptr, "X-Forwarded-For:", 16) == 0) {
+	 && strncasecmp((const char *)packet->line[packet->parsed_lines].ptr, "X-Forwarded-For:", 16) == 0) {
         // some stupid clients omit a space and place the hostname directly after the colon
         if(packet->line[packet->parsed_lines].ptr[16] == ' ') {
           packet->forwarded_line.ptr = &packet->line[packet->parsed_lines].ptr[17];
@@ -5098,58 +5105,58 @@ void ndpi_parse_packet_line_info(struct ndpi_detection_module_struct *ndpi_struc
       }
       /* "Content-Type:" header line in HTTP. */
       if(packet->line[packet->parsed_lines].len > 14
-	     && (strncasecmp((const char *)packet->line[packet->parsed_lines].ptr, "Content-Type: ", 14) == 0
-	         || strncasecmp((const char *)packet->line[packet->parsed_lines].ptr, "Content-type: ", 14) == 0)) {
+	 && (strncasecmp((const char *)packet->line[packet->parsed_lines].ptr, "Content-Type: ", 14) == 0
+	     || strncasecmp((const char *)packet->line[packet->parsed_lines].ptr, "Content-type: ", 14) == 0)) {
         packet->content_line.ptr = &packet->line[packet->parsed_lines].ptr[14];
         packet->content_line.len = packet->line[packet->parsed_lines].len - 14;
         packet->http_num_headers++;
       }
       /* "Content-Type:" header line in HTTP AGAIN. Probably a bogus response without space after ":" */
       if(packet->line[packet->parsed_lines].len > 13
-	     && strncasecmp((const char *)packet->line[packet->parsed_lines].ptr, "Content-type:", 13) == 0) {
+	 && strncasecmp((const char *)packet->line[packet->parsed_lines].ptr, "Content-type:", 13) == 0) {
         packet->content_line.ptr = &packet->line[packet->parsed_lines].ptr[13];
         packet->content_line.len = packet->line[packet->parsed_lines].len - 13;
         packet->http_num_headers++;
       }
       /* "Accept:" header line in HTTP request. */
       if(packet->line[packet->parsed_lines].len > 8
-	     && strncasecmp((const char *)packet->line[packet->parsed_lines].ptr, "Accept: ", 8) == 0) {
+	 && strncasecmp((const char *)packet->line[packet->parsed_lines].ptr, "Accept: ", 8) == 0) {
         packet->accept_line.ptr = &packet->line[packet->parsed_lines].ptr[8];
         packet->accept_line.len = packet->line[packet->parsed_lines].len - 8;
         packet->http_num_headers++;
       }
       /* "Referer:" header line in HTTP request. */
       if(packet->line[packet->parsed_lines].len > 9
-	     && strncasecmp((const char *)packet->line[packet->parsed_lines].ptr, "Referer: ", 9) == 0) {
+	 && strncasecmp((const char *)packet->line[packet->parsed_lines].ptr, "Referer: ", 9) == 0) {
         packet->referer_line.ptr = &packet->line[packet->parsed_lines].ptr[9];
         packet->referer_line.len = packet->line[packet->parsed_lines].len - 9;
         packet->http_num_headers++;
       }
       /* "User-Agent:" header line in HTTP request. */
       if(packet->line[packet->parsed_lines].len > 12
-	     && (strncasecmp((const char *)packet->line[packet->parsed_lines].ptr, "User-Agent: ", 12) == 0
-	         || strncasecmp((const char *)packet->line[packet->parsed_lines].ptr, "User-agent: ", 12) == 0)) {
+	 && (strncasecmp((const char *)packet->line[packet->parsed_lines].ptr, "User-Agent: ", 12) == 0
+	     || strncasecmp((const char *)packet->line[packet->parsed_lines].ptr, "User-agent: ", 12) == 0)) {
         packet->user_agent_line.ptr = &packet->line[packet->parsed_lines].ptr[12];
         packet->user_agent_line.len = packet->line[packet->parsed_lines].len - 12;
         packet->http_num_headers++;
       }
       /* "Content-Encoding:" header line in HTTP response (and request?). */
       if(packet->line[packet->parsed_lines].len > 18
-	     && strncasecmp((const char *)packet->line[packet->parsed_lines].ptr, "Content-Encoding: ", 18) == 0) {
+	 && strncasecmp((const char *)packet->line[packet->parsed_lines].ptr, "Content-Encoding: ", 18) == 0) {
         packet->http_encoding.ptr = &packet->line[packet->parsed_lines].ptr[18];
         packet->http_encoding.len = packet->line[packet->parsed_lines].len - 18;
         packet->http_num_headers++;
       }
       /* "Transfer-Encoding:" header line in HTTP. */
       if(packet->line[packet->parsed_lines].len > 19
-	     && strncasecmp((const char *)packet->line[packet->parsed_lines].ptr, "Transfer-Encoding: ", 19) == 0) {
+	 && strncasecmp((const char *)packet->line[packet->parsed_lines].ptr, "Transfer-Encoding: ", 19) == 0) {
         packet->http_transfer_encoding.ptr = &packet->line[packet->parsed_lines].ptr[19];
         packet->http_transfer_encoding.len = packet->line[packet->parsed_lines].len - 19;
         packet->http_num_headers++;
       }
       /* "Content-Length:" header line in HTTP. */
       if(packet->line[packet->parsed_lines].len > 16
-	     && ((strncasecmp((const char *)packet->line[packet->parsed_lines].ptr, "Content-Length: ", 16) == 0)
+	 && ((strncasecmp((const char *)packet->line[packet->parsed_lines].ptr, "Content-Length: ", 16) == 0)
 	     || (strncasecmp((const char *)packet->line[packet->parsed_lines].ptr, "content-length: ", 16) == 0))) {
         packet->http_contentlen.ptr = &packet->line[packet->parsed_lines].ptr[16];
         packet->http_contentlen.len = packet->line[packet->parsed_lines].len - 16;
@@ -5157,21 +5164,21 @@ void ndpi_parse_packet_line_info(struct ndpi_detection_module_struct *ndpi_struc
       }
       /* "Cookie:" header line in HTTP. */
       if(packet->line[packet->parsed_lines].len > 8
-	     && strncasecmp((const char *)packet->line[packet->parsed_lines].ptr, "Cookie: ", 8) == 0) {
+	 && strncasecmp((const char *)packet->line[packet->parsed_lines].ptr, "Cookie: ", 8) == 0) {
         packet->http_cookie.ptr = &packet->line[packet->parsed_lines].ptr[8];
         packet->http_cookie.len = packet->line[packet->parsed_lines].len - 8;
         packet->http_num_headers++;
       }
       /* "Origin:" header line in HTTP. */
       if(packet->line[packet->parsed_lines].len > 8
-	     && strncasecmp((const char *)packet->line[packet->parsed_lines].ptr, "Origin: ", 8) == 0) {
+	 && strncasecmp((const char *)packet->line[packet->parsed_lines].ptr, "Origin: ", 8) == 0) {
         packet->http_origin.ptr = &packet->line[packet->parsed_lines].ptr[8];
         packet->http_origin.len = packet->line[packet->parsed_lines].len - 8;
         packet->http_num_headers++;
       }
       /* "X-Session-Type:" header line in HTTP. */
       if(packet->line[packet->parsed_lines].len > 16
-	     && strncasecmp((const char *)packet->line[packet->parsed_lines].ptr, "X-Session-Type: ", 16) == 0) {
+	 && strncasecmp((const char *)packet->line[packet->parsed_lines].ptr, "X-Session-Type: ", 16) == 0) {
         packet->http_x_session_type.ptr = &packet->line[packet->parsed_lines].ptr[16];
         packet->http_x_session_type.len = packet->line[packet->parsed_lines].len - 16;
         packet->http_num_headers++;
@@ -5773,7 +5780,7 @@ ndpi_protocol ndpi_guess_undetected_protocol(struct ndpi_detection_module_struct
 /* ****************************************************** */
 
 char* ndpi_protocol2id(struct ndpi_detection_module_struct *ndpi_mod,
-			 ndpi_protocol proto, char *buf, u_int buf_len) {
+		       ndpi_protocol proto, char *buf, u_int buf_len) {
   if((proto.master_protocol != NDPI_PROTOCOL_UNKNOWN)
      && (proto.master_protocol != proto.app_protocol)) {
     if(proto.app_protocol != NDPI_PROTOCOL_UNKNOWN)
@@ -5894,7 +5901,7 @@ ndpi_protocol_category_t ndpi_get_proto_category(struct ndpi_detection_module_st
     return proto.category;
   /* simple rule: sub protocol first, master after */
   else if((proto.master_protocol == NDPI_PROTOCOL_UNKNOWN) ||
-      (ndpi_mod->proto_defaults[proto.app_protocol].protoCategory != NDPI_PROTOCOL_CATEGORY_UNSPECIFIED))
+	  (ndpi_mod->proto_defaults[proto.app_protocol].protoCategory != NDPI_PROTOCOL_CATEGORY_UNSPECIFIED))
     return ndpi_mod->proto_defaults[proto.app_protocol].protoCategory;
   else
     return ndpi_mod->proto_defaults[proto.master_protocol].protoCategory;
@@ -6113,14 +6120,14 @@ static int ndpi_automa_match_string_subprotocol(struct ndpi_detection_module_str
   /*
     TODO HYPERSCAN
     In case of match fill up ret_match and set flow protocol + category
-   */
+  */
   status = hs_scan(hs->database, string_to_match,
 		   string_to_match_len, 0, hs->scratch,
 		   hyperscanEventHandler, &matching_protocol_id);
 
   if(status == HS_SUCCESS) {
     NDPI_LOG_DBG2(ndpi_struct, "[NDPI] Hyperscan engine completed normally. Result: %s [%d][%s]\n",
-		 ndpi_get_proto_name(ndpi_struct, matching_protocol_id), matching_protocol_id, string_to_match);
+		  ndpi_get_proto_name(ndpi_struct, matching_protocol_id), matching_protocol_id, string_to_match);
   } else if(status == HS_SCAN_TERMINATED) {
     NDPI_LOG_DBG2(ndpi_struct, "[NDPI] Hyperscan engine was terminated by callback. Result: %s [%d][%s]\n",
 		  ndpi_get_proto_name(ndpi_struct, matching_protocol_id), matching_protocol_id, string_to_match);
@@ -6142,7 +6149,7 @@ static int ndpi_automa_match_string_subprotocol(struct ndpi_detection_module_str
     m[len] = '\0';
 
     NDPI_LOG_DBG2(ndpi_struct, "[NDPI] ndpi_match_host_subprotocol(%s): %s\n",
-	   m, ndpi_struct->proto_defaults[matching_protocol_id].protoName);
+		  m, ndpi_struct->proto_defaults[matching_protocol_id].protoName);
   }
 #endif
 
@@ -6355,7 +6362,7 @@ void ndpi_lru_add_to_cache(struct ndpi_lru_cache *c, u_int32_t key) {
   - Leave fields empty/zero when information is missing (e.g. with ICMP ports are zero)
   - The hash_buf most be 30+1 bits or longer
   - Return code: 0 = OK, -1 otherwise
- */
+*/
 
 int ndpi_flowv4_flow_hash(u_int8_t l4_proto, u_int32_t src_ip,
 			  u_int32_t dst_ip, u_int16_t src_port, u_int16_t dst_port,
-- 
cgit v1.2.3


From 796472cf846142d07b5d7300d3d70062d25fc0a2 Mon Sep 17 00:00:00 2001
From: emanuele-f <black.silver@hotmail.it>
Date: Fri, 5 Apr 2019 10:11:17 +0200
Subject: Fix category detection due to missing initialization

---
 src/lib/ndpi_main.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'src/lib/ndpi_main.c')

diff --git a/src/lib/ndpi_main.c b/src/lib/ndpi_main.c
index bdd28f666..5d8e21382 100644
--- a/src/lib/ndpi_main.c
+++ b/src/lib/ndpi_main.c
@@ -4731,7 +4731,7 @@ ndpi_protocol ndpi_detection_process_packet(struct ndpi_detection_module_struct
     flow->guessed_host_protocol_id = ndpi_guess_host_protocol_id(ndpi_struct, flow);
 
     if(ndpi_struct->custom_categories.categories_loaded && flow->packet.iph) {
-      ndpi_protocol ret;
+      ndpi_protocol ret = { NDPI_PROTOCOL_UNKNOWN, NDPI_PROTOCOL_UNKNOWN, NDPI_PROTOCOL_CATEGORY_UNSPECIFIED };
       
       ndpi_fill_ip_protocol_category(ndpi_struct, flow->packet.iph->saddr, flow->packet.iph->daddr, &ret);
       flow->guessed_header_category = ret.category;
-- 
cgit v1.2.3