Add support for flow client/server information (#1671)

In a lot of places in ndPI we use *packet* source/dest info
(address/port/direction) when we are interested in *flow* client/server
info, instead.

Add basic logic to autodetect this kind of information.

nDPI doesn't perform any "flow management" itself but this task is
delegated to the external application. It is then likely that the
application might provide more reliable hints about flow
client/server direction and about the TCP handshake presence: in that case,
these information might be (optionally) passed to the library, disabling
the internal "autodetect" logic.

These new fields have been used in some LRU caches and in the "guessing"
algorithm.
It is quite likely that some other code needs to be updated.

9 files changed, 223 insertions, 129 deletions

diff --git a/src/include/ndpi_api.h.in b/src/include/ndpi_api.h.in
index 2db7a4c0e..a069d0571 100644
--- a/src/include/ndpi_api.h.in
+++ b/src/include/ndpi_api.h.in
@@ -302,6 +302,7 @@ extern "C" {
    * @par    packet         = unsigned char pointer to the Layer 3 (IP header)
    * @par    packetlen      = the length of the packet
    * @par    packet_time_ms = the current timestamp for the packet (expressed in msec)
+   * @par    input_info     = (optional) flow information provided by the (external) flow manager
    * @return void
    *
    */
@@ -309,7 +310,8 @@ extern "C" {
 				 struct ndpi_flow_struct *flow,
 				 const unsigned char *packet,
 				 const unsigned short packetlen,
-				 const u_int64_t packet_time_ms);
+				 const u_int64_t packet_time_ms,
+				 const struct ndpi_flow_input_info *input_info);
 
   /**
    * Processes one packet and returns the ID of the detected protocol.
@@ -320,6 +322,7 @@ extern "C" {
    * @par    packet         = unsigned char pointer to the Layer 3 (IP header)
    * @par    packetlen      = the length of the packet
    * @par    packet_time_ms = the current timestamp for the packet (expressed in msec)
+   * @par    input_info     = (optional) flow information provided by the (external) flow manager
    * @return the detected ID of the protocol
    *
    */
@@ -327,7 +330,8 @@ extern "C" {
 					      struct ndpi_flow_struct *flow,
 					      const unsigned char *packet,
 					      const unsigned short packetlen,
-					      const u_int64_t packet_time_ms);
+					      const u_int64_t packet_time_ms,
+					      const struct ndpi_flow_input_info *input_info);
   /**
    * Get the main protocol of the passed flows for the detected module
    *
diff --git a/src/include/ndpi_main.h b/src/include/ndpi_main.h
index 071097d99..355b9a086 100644
--- a/src/include/ndpi_main.h
+++ b/src/include/ndpi_main.h
@@ -167,6 +167,10 @@ extern "C" {
 
   int64_t ndpi_asn1_ber_decode_length(const unsigned char *payload, int payload_len, u_int16_t *value_len);
 
+  int ndpi_current_pkt_from_client_to_server(const struct ndpi_packet_struct *packet, const struct ndpi_flow_struct *flow);
+  int ndpi_current_pkt_from_server_to_client(const struct ndpi_packet_struct *packet, const struct ndpi_flow_struct *flow);
+  int ndpi_seen_flow_beginning(const struct ndpi_flow_struct *flow);
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/src/include/ndpi_typedefs.h b/src/include/ndpi_typedefs.h
index bcbdab296..9928ce321 100644
--- a/src/include/ndpi_typedefs.h
+++ b/src/include/ndpi_typedefs.h
@@ -528,6 +528,31 @@ struct ndpi_vxlanhdr {
 } PACK_OFF;
 
 /* ************************************************************ */
+
+/**
+ * The application might inform the library about client/server direction
+ */
+#define NDPI_IN_PKT_DIR_UNKNOWN		0	/**< The application doesn't provide this kind of information */
+#define NDPI_IN_PKT_DIR_C_TO_S		1	/**< Current packet is from client to server */
+#define NDPI_IN_PKT_DIR_S_TO_C		2	/**< Current packet is from server to client */
+
+/**
+ * The application might choose to not pass TCP handshake packets to the library
+ * (for performance reasons), but it might want to inform the library itlsef that these
+ * packets have been captured/seen anyway (to avoid losing classifiation capabilities).
+ */
+#define NDPI_FLOW_BEGINNING_UNKNOWN	0	/**< The application doesn't provide this kind of information */
+#define NDPI_FLOW_BEGINNING_SEEN	1	/**< The application informs the library that the TCP handshake has been seen (even if its packets might not have been passed to the library) */
+#define NDPI_FLOW_BEGINNING_NOT_SEEN	2	/**< The application informs the library that the TCP handshake has not been seen */
+
+/**
+ * Optional information about flow management (per packet)
+ */
+struct ndpi_flow_input_info {
+  unsigned char in_pkt_dir;
+  unsigned char seen_flow_beginning;
+};
+
 /* ******************* ********************* ****************** */
 /* ************************************************************ */
 
@@ -1150,6 +1175,7 @@ struct ndpi_detection_module_struct {
 
   /* Current packet */
   struct ndpi_packet_struct packet;
+  const struct ndpi_flow_input_info *input_info;
 };
 
 #endif /* NDPI_LIB_COMPILATION */
@@ -1182,7 +1208,7 @@ struct ndpi_flow_struct {
   /* init parameter, internal used to set up timestamp,... */
   u_int16_t guessed_protocol_id, guessed_host_protocol_id, guessed_category, guessed_header_category;
   u_int8_t l4_proto, protocol_id_already_guessed:1, host_already_guessed:1, fail_with_unknown:1,
-    init_finished:1, setup_packet_direction:1, packet_direction:1, check_extra_packets:1, is_ipv6:1;
+    init_finished:1, client_packet_direction:1, packet_direction:1, check_extra_packets:1, is_ipv6:1;
   u_int16_t num_dissector_calls;
   ndpi_confidence_t confidence; /* ndpi_confidence_t */
 
@@ -1192,14 +1218,15 @@ struct ndpi_flow_struct {
   */
   u_int32_t next_tcp_seq_nr[2];
 
-  /* Flow addresses (used mainly for LRU lookups in ndpi_detection_giveup())
-     and ports. All in *network* byte order
-
-     TODO
-     - IPv6. Note that LRU is ipv4 only, for the time being 
+  /* Flow addresses (useful for LRU lookups in ndpi_detection_giveup())
+     and ports. All in *network* byte order.
+     Client and server.
    */
-  u_int32_t saddr, daddr;
-  u_int16_t sport, dport;
+  union {
+    u_int32_t v4;
+    u_int8_t v6[16];
+  } c_address, s_address;	/* For some unknown reasons, x86_64-w64-mingw32-gcc doesn't like the name "s_addr" */
+  u_int16_t c_port, s_port;
   
   // -----------------------------------------
 
diff --git a/src/lib/ndpi_main.c b/src/lib/ndpi_main.c
index afa01d8fd..0ef07454e 100644
--- a/src/lib/ndpi_main.c
+++ b/src/lib/ndpi_main.c
@@ -2156,6 +2156,19 @@ u_int16_t ndpi_network_port_ptree_match(struct ndpi_detection_module_struct *ndp
   ndpi_prefix_t prefix;
   ndpi_patricia_node_t *node;
 
+  if(ndpi_str->ndpi_num_custom_protocols == 0) {
+    /*
+      In case we don't have defined any custom protocol we check the ptree
+      only in case of public IP addresses as in ndpi_content_match.c.inc
+      we only have public IP addresses. Instead with custom protocols, users
+      might have defined private protocols hence we should not skip
+      the checks below
+    */
+
+    if(ndpi_is_public_ipv4(ntohl(pin->s_addr)) == 0)
+      return(NDPI_PROTOCOL_UNKNOWN); /* Non public IP */
+  }
+
   /* Make sure all in network byte order otherwise compares wont work */
   ndpi_fill_prefix_v4(&prefix, pin, 32, ((ndpi_patricia_tree_t *) ndpi_str->protocols_ptree)->maxbits);
   node = ndpi_patricia_search_best(ndpi_str->protocols_ptree, &prefix);
@@ -3084,16 +3097,14 @@ static ndpi_default_ports_tree_node_t *ndpi_get_guessed_protocol_id(struct ndpi_
   ndpi_default_ports_tree_node_t node;
 
   if(sport && dport) {
-    int low = ndpi_min(sport, dport);
-    int high = ndpi_max(sport, dport);
     const void *ret;
 
-    node.default_port = low; /* Check server port first */
+    node.default_port = dport; /* Check server port first */
     ret = ndpi_tfind(&node, (proto == IPPROTO_TCP) ? (void *) &ndpi_str->tcpRoot : (void *) &ndpi_str->udpRoot,
 		     ndpi_default_ports_tree_node_t_cmp);
 
     if(ret == NULL) {
-      node.default_port = high;
+      node.default_port = sport;
       ret = ndpi_tfind(&node, (proto == IPPROTO_TCP) ? (void *) &ndpi_str->tcpRoot : (void *) &ndpi_str->udpRoot,
 		       ndpi_default_ports_tree_node_t_cmp);
     }
@@ -4850,7 +4861,8 @@ static int ndpi_init_packet(struct ndpi_detection_module_struct *ndpi_str,
 			    struct ndpi_flow_struct *flow,
 			    const u_int64_t current_time_ms,
 			    const unsigned char *packet_data,
-			    unsigned short packetlen) {
+			    unsigned short packetlen,
+			    const struct ndpi_flow_input_info *input_info) {
   struct ndpi_packet_struct *packet = &ndpi_str->packet;
   const struct ndpi_iphdr *decaps_iph = NULL;
   u_int16_t l3len;
@@ -4868,6 +4880,8 @@ static int ndpi_init_packet(struct ndpi_detection_module_struct *ndpi_str,
 
   packet->current_time_ms = current_time_ms;
 
+  ndpi_str->input_info = input_info;
+
   packet->iph = (const struct ndpi_iphdr *)packet_data;
 
   /* reset payload_packet_len, will be set if ipv4 tcp or udp */
@@ -5040,36 +5054,32 @@ void ndpi_connection_tracking(struct ndpi_detection_module_struct *ndpi_str,
     }
 
     flow->is_ipv6 = (packet->iphv6 != NULL);
-    if(flow->is_ipv6 == 0)
-      flow->saddr = packet->iph->saddr, flow->daddr = packet->iph->daddr; /* See (*#*) */
 
     flow->last_packet_time_ms = packet->current_time_ms;
 
     packet->packet_lines_parsed_complete = 0;
 
-    if(flow->init_finished == 0) {
-      flow->init_finished = 1;
-      flow->setup_packet_direction = packet->packet_direction;
-    }
-
     if(tcph != NULL) {
-      flow->sport = tcph->source, flow->dport = tcph->dest; /* (*#*) */
-
       if(!ndpi_str->direction_detect_disable)
 	packet->packet_direction = (ntohs(tcph->source) < ntohs(tcph->dest)) ? 1 : 0;
 
-      if(tcph->syn != 0 && tcph->ack == 0 && flow->l4.tcp.seen_syn == 0 && flow->l4.tcp.seen_syn_ack == 0 &&
-	 flow->l4.tcp.seen_ack == 0) {
-	flow->l4.tcp.seen_syn = 1;
-      } else
-	if(tcph->syn != 0 && tcph->ack != 0 && flow->l4.tcp.seen_syn == 1 && flow->l4.tcp.seen_syn_ack == 0 &&
-	   flow->l4.tcp.seen_ack == 0) {
-	  flow->l4.tcp.seen_syn_ack = 1;
-	} else
-	  if(tcph->syn == 0 && tcph->ack == 1 && flow->l4.tcp.seen_syn == 1 && flow->l4.tcp.seen_syn_ack == 1 &&
-	     flow->l4.tcp.seen_ack == 0) {
-	    flow->l4.tcp.seen_ack = 1;
-	  }
+      if(ndpi_str->input_info == NULL ||
+         ndpi_str->input_info->seen_flow_beginning == NDPI_FLOW_BEGINNING_UNKNOWN) {
+        if(tcph->syn != 0 && tcph->ack == 0 && flow->l4.tcp.seen_syn == 0 && flow->l4.tcp.seen_syn_ack == 0 &&
+           flow->l4.tcp.seen_ack == 0) {
+          flow->l4.tcp.seen_syn = 1;
+        } else {
+          if(tcph->syn != 0 && tcph->ack != 0 && flow->l4.tcp.seen_syn == 1 && flow->l4.tcp.seen_syn_ack == 0 &&
+             flow->l4.tcp.seen_ack == 0) {
+            flow->l4.tcp.seen_syn_ack = 1;
+          } else {
+            if(tcph->syn == 0 && tcph->ack == 1 && flow->l4.tcp.seen_syn == 1 && flow->l4.tcp.seen_syn_ack == 1 &&
+               flow->l4.tcp.seen_ack == 0) {
+              flow->l4.tcp.seen_ack = 1;
+            }
+          }
+        }
+      }
 
       if((flow->next_tcp_seq_nr[0] == 0 && flow->next_tcp_seq_nr[1] == 0) ||
 	 (flow->next_tcp_seq_nr[0] == 0 || flow->next_tcp_seq_nr[1] == 0)) {
@@ -5115,12 +5125,79 @@ void ndpi_connection_tracking(struct ndpi_detection_module_struct *ndpi_str,
 	flow->next_tcp_seq_nr[1] = 0;
       }
     } else if(udph != NULL) {
-      flow->sport = udph->source, flow->dport = udph->dest; /* (*#*) */
-
       if(!ndpi_str->direction_detect_disable)
 	packet->packet_direction = (htons(udph->source) < htons(udph->dest)) ? 1 : 0;
     }
 
+    if(flow->init_finished == 0) {
+      u_int16_t s_port, d_port; /* Source/Dest ports */
+
+      flow->init_finished = 1;
+
+      if(ndpi_str->input_info &&
+         ndpi_str->input_info->seen_flow_beginning == NDPI_FLOW_BEGINNING_SEEN) {
+        flow->l4.tcp.seen_syn = 1;
+        flow->l4.tcp.seen_syn_ack = 1;
+        flow->l4.tcp.seen_ack = 1;
+      }
+
+      /* Client/Server direction */
+
+      s_port = 0;
+      d_port = 0;
+      if(tcph != NULL) {
+        s_port = tcph->source;
+        d_port = tcph->dest;
+      } else if(udph != NULL) {
+        s_port = udph->source;
+        d_port = udph->dest;
+      }
+
+      if(ndpi_str->input_info &&
+         ndpi_str->input_info->in_pkt_dir != NDPI_IN_PKT_DIR_UNKNOWN) {
+        if(ndpi_str->input_info->in_pkt_dir == NDPI_IN_PKT_DIR_C_TO_S)
+          flow->client_packet_direction = packet->packet_direction;
+        else
+          flow->client_packet_direction = !packet->packet_direction;
+      } else {
+        if(tcph && tcph->syn) {
+          if(tcph->ack == 0) {
+            flow->client_packet_direction = packet->packet_direction;
+          } else {
+            flow->client_packet_direction = !packet->packet_direction;
+          }
+        } else if(ntohs(s_port) > 1024 && ntohs(d_port) < 1024) {
+          flow->client_packet_direction = packet->packet_direction;
+        } else if(ntohs(s_port) < 1024 && ntohs(d_port) > 1024) {
+          flow->client_packet_direction = !packet->packet_direction;
+        } else {
+          flow->client_packet_direction = packet->packet_direction;
+        }
+      }
+
+      if(ndpi_current_pkt_from_client_to_server(packet, flow)) {
+        if(flow->is_ipv6 == 0) {
+          flow->c_address.v4 = packet->iph->saddr;
+          flow->s_address.v4 = packet->iph->daddr;
+        } else {
+          memcpy(flow->c_address.v6, &packet->iphv6->ip6_src, 16);
+          memcpy(flow->s_address.v6, &packet->iphv6->ip6_dst, 16);
+        }
+        flow->c_port = s_port;
+        flow->s_port = d_port;
+      } else {
+        if(flow->is_ipv6 == 0) {
+          flow->c_address.v4 = packet->iph->daddr;
+          flow->s_address.v4 = packet->iph->saddr;
+        } else {
+          memcpy(flow->c_address.v6, &packet->iphv6->ip6_dst, 16);
+          memcpy(flow->s_address.v6, &packet->iphv6->ip6_src, 16);
+        }
+        flow->c_port = d_port;
+        flow->s_port = s_port;
+      }
+    }
+
     if(flow->packet_counter < MAX_PACKET_COUNTER && packet->payload_packet_len) {
       flow->packet_counter++;
     }
@@ -5319,23 +5396,14 @@ u_int16_t ndpi_guess_host_protocol_id(struct ndpi_detection_module_struct *ndpi_
 
   if(packet->iph) {
     struct in_addr addr;
-    u_int16_t sport, dport;
 
-    addr.s_addr = packet->iph->saddr;
-
-    if((flow->l4_proto == IPPROTO_TCP) && packet->tcp)
-      sport = packet->tcp->source, dport = packet->tcp->dest;
-    else if((flow->l4_proto == IPPROTO_UDP) && packet->udp)
-      sport = packet->udp->source, dport = packet->udp->dest;
-    else
-      sport = dport = 0;
-
-    /* guess host protocol */
-    ret = ndpi_network_port_ptree_match(ndpi_str, &addr, sport);
+    /* guess host protocol; server first */
+    addr.s_addr = flow->s_address.v4;
+    ret = ndpi_network_port_ptree_match(ndpi_str, &addr, ntohs(flow->s_port));
 
     if(ret == NDPI_PROTOCOL_UNKNOWN) {
-      addr.s_addr = packet->iph->daddr;
-      ret = ndpi_network_port_ptree_match(ndpi_str, &addr, dport);
+      addr.s_addr = flow->c_address.v4;
+      ret = ndpi_network_port_ptree_match(ndpi_str, &addr, ntohs(flow->c_port));
     }
   }
 
@@ -5374,7 +5442,7 @@ static void ndpi_reconcile_protocols(struct ndpi_detection_module_struct *ndpi_s
 
       if(ndpi_str->msteams_cache)
 	ndpi_lru_add_to_cache(ndpi_str->msteams_cache,
-			      ntohl(flow->saddr),
+			      ntohl(flow->c_address.v4),
 			      (flow->last_packet_time_ms / 1000) & 0xFFFF /* 16 bit */);
     }
     break;
@@ -5386,7 +5454,7 @@ static void ndpi_reconcile_protocols(struct ndpi_detection_module_struct *ndpi_s
        && ndpi_str->msteams_cache) {
       u_int16_t when;
 
-      if(ndpi_lru_find_cache(ndpi_str->msteams_cache, ntohl(flow->saddr),
+      if(ndpi_lru_find_cache(ndpi_str->msteams_cache, ntohl(flow->c_address.v4),
 			     &when, 0 /* Don't remove it as it can be used for other connections */)) {
 	u_int16_t tdiff = ((flow->last_packet_time_ms /1000) & 0xFFFF) - when;
 
@@ -5396,7 +5464,7 @@ static void ndpi_reconcile_protocols(struct ndpi_detection_module_struct *ndpi_s
 
 	  /* Refresh cache */
 	  ndpi_lru_add_to_cache(ndpi_str->msteams_cache,
-				ntohl(flow->saddr),
+				ntohl(flow->c_address.v4),
 				(flow->last_packet_time_ms / 1000) & 0xFFFF /* 16 bit */);
 	}
       }
@@ -5547,7 +5615,7 @@ ndpi_protocol ndpi_detection_giveup(struct ndpi_detection_module_struct *ndpi_st
   if(ndpi_str->mining_cache && flow->is_ipv6 == 0) {
     u_int16_t cached_proto;
 
-    if(ndpi_lru_find_cache(ndpi_str->mining_cache, flow->saddr + flow->daddr,
+    if(ndpi_lru_find_cache(ndpi_str->mining_cache, flow->c_address.v4 + flow->s_address.v4,
 			   &cached_proto, 0 /* Don't remove it as it can be used for other connections */)) {
       ndpi_set_detected_protocol(ndpi_str, flow, cached_proto, NDPI_PROTOCOL_UNKNOWN, NDPI_CONFIDENCE_DPI_PARTIAL_CACHE);
       ret.master_protocol = flow->detected_protocol_stack[1], ret.app_protocol = flow->detected_protocol_stack[0];
@@ -5660,14 +5728,14 @@ ndpi_protocol ndpi_detection_giveup(struct ndpi_detection_module_struct *ndpi_st
      && (ret.app_protocol == NDPI_PROTOCOL_UNKNOWN)) {
     /* Last resort */
     if(ndpi_search_into_bittorrent_cache(ndpi_str, flow,
-					 flow->saddr, flow->sport,
-					 flow->daddr, flow->dport)) {
+					 flow->c_address.v4, flow->c_port,
+					 flow->s_address.v4, flow->s_port)) {
       /* This looks like BitTorrent */
       ndpi_set_detected_protocol(ndpi_str, flow, NDPI_PROTOCOL_BITTORRENT, NDPI_PROTOCOL_UNKNOWN, NDPI_CONFIDENCE_DPI_PARTIAL_CACHE);
       ret.app_protocol = NDPI_PROTOCOL_BITTORRENT;
     } else if((flow->l4_proto == IPPROTO_UDP) /* Zoom/UDP used for video */
-	      && (((ntohs(flow->sport) == 8801 /* Zoom port */) && ndpi_search_into_zoom_cache(ndpi_str, flow->saddr))
-		  || ((ntohs(flow->dport) == 8801 /* Zoom port */) && ndpi_search_into_zoom_cache(ndpi_str, flow->daddr))
+	      && (((ntohs(flow->c_port) == 8801 /* Zoom port */) && ndpi_search_into_zoom_cache(ndpi_str, flow->c_address.v4))
+		  || ((ntohs(flow->s_port) == 8801 /* Zoom port */) && ndpi_search_into_zoom_cache(ndpi_str, flow->s_address.v4))
 		  )) {
       /* This looks like Zoom */
       ndpi_set_detected_protocol(ndpi_str, flow, NDPI_PROTOCOL_ZOOM, NDPI_PROTOCOL_UNKNOWN, NDPI_CONFIDENCE_DPI_PARTIAL_CACHE);
@@ -5688,12 +5756,13 @@ ndpi_protocol ndpi_detection_giveup(struct ndpi_detection_module_struct *ndpi_st
 
 void ndpi_process_extra_packet(struct ndpi_detection_module_struct *ndpi_str, struct ndpi_flow_struct *flow,
 			       const unsigned char *packet_data, const unsigned short packetlen,
-			       const u_int64_t current_time_ms) {
+			       const u_int64_t current_time_ms,
+			       const struct ndpi_flow_input_info *input_info) {
   if(flow == NULL)
     return;
 
   /* set up the packet headers for the extra packet function to use if it wants */
-  if(ndpi_init_packet(ndpi_str, flow, current_time_ms, packet_data, packetlen) != 0)
+  if(ndpi_init_packet(ndpi_str, flow, current_time_ms, packet_data, packetlen, input_info) != 0)
     return;
 
   ndpi_connection_tracking(ndpi_str, flow);
@@ -5977,29 +6046,15 @@ static int ndpi_do_guess(struct ndpi_detection_module_struct *ndpi_str, struct n
   ret->master_protocol = ret->app_protocol = NDPI_PROTOCOL_UNKNOWN, ret->category = 0;
 
   if(packet->iphv6 || packet->iph) {
-    u_int16_t sport, dport;
-    u_int8_t protocol;
     u_int8_t user_defined_proto;
 
-    if(packet->iphv6 != NULL) {
-      protocol = packet->iphv6->ip6_hdr.ip6_un1_nxt;
-    } else
-      protocol = packet->iph->protocol;
-
-    if(packet->udp)
-      sport = ntohs(packet->udp->source), dport = ntohs(packet->udp->dest);
-    else if(packet->tcp)
-      sport = ntohs(packet->tcp->source), dport = ntohs(packet->tcp->dest);
-    else
-      sport = dport = 0;
-
     /* guess protocol */
-    flow->guessed_protocol_id      = (int16_t) ndpi_guess_protocol_id(ndpi_str, flow, protocol, sport, dport, &user_defined_proto);
+    flow->guessed_protocol_id      = (int16_t) ndpi_guess_protocol_id(ndpi_str, flow, flow->l4_proto, ntohs(flow->c_port), ntohs(flow->s_port), &user_defined_proto);
     flow->guessed_host_protocol_id = ndpi_guess_host_protocol_id(ndpi_str, flow);
 
     if(ndpi_str->custom_categories.categories_loaded && packet->iph) {
       if(ndpi_str->ndpi_num_custom_protocols != 0)
-	ndpi_fill_ip_protocol_category(ndpi_str, packet->iph->saddr, packet->iph->daddr, ret);
+	ndpi_fill_ip_protocol_category(ndpi_str, flow->c_address.v4, flow->s_address.v4, ret);
       flow->guessed_header_category = ret->category;
     } else
       flow->guessed_header_category = NDPI_PROTOCOL_CATEGORY_UNSPECIFIED;
@@ -6007,9 +6062,7 @@ static int ndpi_do_guess(struct ndpi_detection_module_struct *ndpi_str, struct n
     if(flow->guessed_protocol_id >= NDPI_MAX_SUPPORTED_PROTOCOLS) {
       /* This is a custom protocol and it has priority over everything else */
       ret->master_protocol = NDPI_PROTOCOL_UNKNOWN,
-	ret->app_protocol = flow->guessed_protocol_id ? flow->guessed_protocol_id : flow->guessed_host_protocol_id;
-
-      // if(ndpi_str->ndpi_num_custom_protocols != 0)
+      ret->app_protocol = flow->guessed_protocol_id ? flow->guessed_protocol_id : flow->guessed_host_protocol_id;
       flow->confidence = NDPI_CONFIDENCE_MATCH_BY_PORT; /* TODO */
       ndpi_fill_protocol_category(ndpi_str, flow, ret);
       return(-1);
@@ -6063,7 +6116,6 @@ static int ndpi_do_guess(struct ndpi_detection_module_struct *ndpi_str, struct n
 
     flow->num_dissector_calls += ndpi_check_flow_func(ndpi_str, flow, &ndpi_selection_packet);
 
-    //if(ndpi_str->ndpi_num_custom_protocols != 0)
     ndpi_fill_protocol_category(ndpi_str, flow, ret);
     return(-1);
   }
@@ -6075,7 +6127,8 @@ static int ndpi_do_guess(struct ndpi_detection_module_struct *ndpi_str, struct n
 
 ndpi_protocol ndpi_detection_process_packet(struct ndpi_detection_module_struct *ndpi_str,
 					    struct ndpi_flow_struct *flow, const unsigned char *packet_data,
-					    const unsigned short packetlen, const u_int64_t current_time_ms) {
+					    const unsigned short packetlen, const u_int64_t current_time_ms,
+					    const struct ndpi_flow_input_info *input_info) {
   struct ndpi_packet_struct *packet = &ndpi_str->packet;
   NDPI_SELECTION_BITMASK_PROTOCOL_SIZE ndpi_selection_packet;
   u_int32_t num_calls = 0;
@@ -6108,19 +6161,19 @@ ndpi_protocol ndpi_detection_process_packet(struct ndpi_detection_module_struct
     ret.app_protocol = flow->detected_protocol_stack[0];
 
   if(flow->check_extra_packets) {
-    ndpi_process_extra_packet(ndpi_str, flow, packet_data, packetlen, current_time_ms);
+    ndpi_process_extra_packet(ndpi_str, flow, packet_data, packetlen, current_time_ms, input_info);
     /* Update in case of new match */
     ret.master_protocol = flow->detected_protocol_stack[1],
       ret.app_protocol = flow->detected_protocol_stack[0],
       ret.category = flow->category;
     return ret;
   } else if(flow->detected_protocol_stack[0] != NDPI_PROTOCOL_UNKNOWN) {
-    if(ndpi_init_packet(ndpi_str, flow, current_time_ms, packet_data, packetlen) != 0)
+    if(ndpi_init_packet(ndpi_str, flow, current_time_ms, packet_data, packetlen, input_info) != 0)
       return ret;
     goto ret_protocols;
   }
 
-  if(ndpi_init_packet(ndpi_str, flow, current_time_ms, packet_data, packetlen) != 0)
+  if(ndpi_init_packet(ndpi_str, flow, current_time_ms, packet_data, packetlen, input_info) != 0)
     return ret;
 
   ndpi_connection_tracking(ndpi_str, flow);
@@ -6200,20 +6253,20 @@ ndpi_protocol ndpi_detection_process_packet(struct ndpi_detection_module_struct
      && ((ret.master_protocol != NDPI_PROTOCOL_UNKNOWN) || (ret.app_protocol != NDPI_PROTOCOL_UNKNOWN))
      ) {
     ndpi_default_ports_tree_node_t *found;
-    u_int16_t *default_ports, sport, dport;
+    u_int16_t *default_ports;
 
     if(packet->udp)
       found = ndpi_get_guessed_protocol_id(ndpi_str, IPPROTO_UDP,
-					   sport = ntohs(packet->udp->source),
-					   dport = ntohs(packet->udp->dest)),
+					   ntohs(flow->c_port),
+					   ntohs(flow->s_port)),
 	default_ports = ndpi_str->proto_defaults[ret.master_protocol ? ret.master_protocol : ret.app_protocol].udp_default_ports;
     else if(packet->tcp)
       found = ndpi_get_guessed_protocol_id(ndpi_str, IPPROTO_TCP,
-					   sport = ntohs(packet->tcp->source),
-					   dport = ntohs(packet->tcp->dest)),
+					   ntohs(flow->c_port),
+					   ntohs(flow->s_port)),
 	default_ports = ndpi_str->proto_defaults[ret.master_protocol ? ret.master_protocol : ret.app_protocol].tcp_default_ports;
     else
-      found = NULL, default_ports = NULL, sport = dport = 0;
+      found = NULL, default_ports = NULL;
 
     if(found
        && (found->proto->protoId != NDPI_PROTOCOL_UNKNOWN)
@@ -6231,7 +6284,7 @@ ndpi_protocol ndpi_detection_process_packet(struct ndpi_detection_module_struct
 	u_int8_t found = 0, i;
 
 	for(i=0; (i<MAX_DEFAULT_PORTS) && (default_ports[i] != 0); i++) {
-	  if(default_ports[i] == dport) {
+	  if(default_ports[i] == ntohs(flow->s_port)) {
 	    found = 1;
 	    break;
 	  }
@@ -6239,7 +6292,7 @@ ndpi_protocol ndpi_detection_process_packet(struct ndpi_detection_module_struct
 
 	if(!found) {
 	  ndpi_default_ports_tree_node_t *r = ndpi_get_guessed_protocol_id(ndpi_str, packet->udp ? IPPROTO_UDP : IPPROTO_TCP,
-									   sport, dport);
+									   ntohs(flow->c_port), ntohs(flow->s_port));
 
 	  if((r == NULL)
 	     || ((r->proto->protoId != ret.app_protocol) && (r->proto->protoId != ret.master_protocol))) {
@@ -6270,7 +6323,7 @@ ndpi_protocol ndpi_detection_process_packet(struct ndpi_detection_module_struct
 
     check_default_ports:
       for(i=0; (i<MAX_DEFAULT_PORTS) && (default_ports[i] != 0); i++) {
-	if((default_ports[i] == sport) || (default_ports[i] == dport)) {
+	if((default_ports[i] == ntohs(flow->c_port)) || (default_ports[i] == ntohs(flow->s_port))) {
 	  found = 1;
 	  break;
 	}
@@ -6288,7 +6341,7 @@ ndpi_protocol ndpi_detection_process_packet(struct ndpi_detection_module_struct
 
       if(!found) {
 	ndpi_default_ports_tree_node_t *r = ndpi_get_guessed_protocol_id(ndpi_str, packet->udp ? IPPROTO_UDP : IPPROTO_TCP,
-									   sport, dport);
+									   ntohs(flow->c_port), ntohs(flow->s_port));
 	
 	if((r == NULL)
 	   || ((r->proto->protoId != ret.app_protocol) && (r->proto->protoId != ret.master_protocol)))	  
@@ -8696,3 +8749,30 @@ char *ndpi_user_agent_set(struct ndpi_flow_struct *flow, const u_int8_t *value,
 
   return flow->http.user_agent;
 }
+
+/* ******************************************************************** */
+
+int ndpi_current_pkt_from_client_to_server(const struct ndpi_packet_struct *packet,
+					   const struct ndpi_flow_struct *flow)
+{
+  return packet->packet_direction == flow->client_packet_direction;
+}
+
+/* ******************************************************************** */
+
+int ndpi_current_pkt_from_server_to_client(const struct ndpi_packet_struct *packet,
+					   const struct ndpi_flow_struct *flow)
+{
+  return packet->packet_direction != flow->client_packet_direction;
+}
+
+/* ******************************************************************** */
+
+int ndpi_seen_flow_beginning(const struct ndpi_flow_struct *flow)
+{
+  if(flow->l4_proto == IPPROTO_TCP &&
+     (flow->l4.tcp.seen_syn == 0 || flow->l4.tcp.seen_syn_ack == 0 ||
+      flow->l4.tcp.seen_ack == 0))
+    return 0;
+  return 1;
+}
diff --git a/src/lib/ndpi_utils.c b/src/lib/ndpi_utils.c
index 6f8b4603c..6a58b5982 100644
--- a/src/lib/ndpi_utils.c
+++ b/src/lib/ndpi_utils.c
@@ -2271,8 +2271,8 @@ static void ndpi_handle_risk_exceptions(struct ndpi_detection_module_struct *ndp
   /* TODO: add IPv6 support */
   if(!flow->ip_risk_mask_evaluated) {
     if(flow->is_ipv6 == 0) {
-      ndpi_check_ipv4_exception(ndpi_str, flow, flow->saddr /* Source */);
-      ndpi_check_ipv4_exception(ndpi_str, flow, flow->daddr /* Destination */);
+      ndpi_check_ipv4_exception(ndpi_str, flow, flow->c_address.v4 /* Client */);
+      ndpi_check_ipv4_exception(ndpi_str, flow, flow->s_address.v4 /* Server */);
     }
 
     flow->ip_risk_mask_evaluated = 1;
diff --git a/src/lib/protocols/bittorrent.c b/src/lib/protocols/bittorrent.c
index d2cfa93f5..435dc1089 100644
--- a/src/lib/protocols/bittorrent.c
+++ b/src/lib/protocols/bittorrent.c
@@ -126,34 +126,25 @@ static void ndpi_add_connection_as_bittorrent(struct ndpi_detection_module_struc
   if(ndpi_struct->bittorrent_cache && packet->iph) {
     u_int32_t key1, key2, i;
 
-    if(packet->udp)
-      key1 = ndpi_ip_port_hash_funct(packet->iph->saddr, packet->udp->source), key2 = ndpi_ip_port_hash_funct(packet->iph->daddr, packet->udp->dest);
-    else
-      key1 = ndpi_ip_port_hash_funct(packet->iph->saddr, packet->tcp->source), key2 = ndpi_ip_port_hash_funct(packet->iph->daddr, packet->tcp->dest);
+    key1 = ndpi_ip_port_hash_funct(flow->c_address.v4, flow->c_port), key2 = ndpi_ip_port_hash_funct(flow->s_address.v4, flow->s_port);
     
     ndpi_lru_add_to_cache(ndpi_struct->bittorrent_cache, key1, NDPI_PROTOCOL_BITTORRENT);
     ndpi_lru_add_to_cache(ndpi_struct->bittorrent_cache, key2, NDPI_PROTOCOL_BITTORRENT);
 
     /* Now add hosts as twins */
     ndpi_lru_add_to_cache(ndpi_struct->bittorrent_cache,
-			  packet->iph->saddr + packet->iph->daddr,
+			  flow->c_address.v4 + flow->s_address.v4,
 			  NDPI_PROTOCOL_BITTORRENT);
     
     /* Also add +2 ports of the sender in order to catch additional sockets open by the same client */
     for(i=0; i<2; i++) {
-      if(packet->udp)
-	key1 = ndpi_ip_port_hash_funct(packet->iph->saddr, htons(ntohs(packet->udp->source)+1+i));
-      else
-	key1 = ndpi_ip_port_hash_funct(packet->iph->saddr, htons(ntohs(packet->tcp->source)+1+i));
+      key1 = ndpi_ip_port_hash_funct(flow->c_address.v4, htons(ntohs(flow->c_port)+1+i));
 
       ndpi_lru_add_to_cache(ndpi_struct->bittorrent_cache, key1, NDPI_PROTOCOL_BITTORRENT);
     }
     
 #ifdef BITTORRENT_CACHE_DEBUG
-    if(packet->udp)
-      printf("[BitTorrent] [UDP] *** ADDED ports %u / %u [%u][%u]\n", ntohs(packet->udp->source), ntohs(packet->udp->dest), key1, key2);
-    else
-      printf("[BitTorrent] [TCP] *** ADDED ports %u / %u [%u][%u]\n", ntohs(packet->tcp->source), ntohs(packet->tcp->dest), key1, key2);
+    printf("[BitTorrent] [%s] *** ADDED ports %u / %u [%u][%u]\n", flow->l4_proto == IPPROTO_TCP ? "TCP" : "UDP", ntohs(flow->c_port), ntohs(flow->s_port), key1, key2);
 #endif
   }
 }
@@ -455,14 +446,7 @@ static u_int8_t is_port(u_int16_t a, u_int16_t b, u_int16_t what) {
 static void ndpi_skip_bittorrent(struct ndpi_detection_module_struct *ndpi_struct,
 				 struct ndpi_flow_struct *flow,
 				 struct ndpi_packet_struct *packet) {
-  u_int16_t sport, dport;
-
-  if(packet->udp)
-    sport = packet->udp->source, dport = packet->udp->dest;
-  else
-    sport = packet->tcp->source, dport = packet->tcp->dest;
-  
-  if(packet->iph && ndpi_search_into_bittorrent_cache(ndpi_struct, flow, packet->iph->saddr, sport, packet->iph->daddr, dport))
+  if(packet->iph && ndpi_search_into_bittorrent_cache(ndpi_struct, flow, flow->c_address.v4, flow->c_port, flow->s_address.v4, flow->s_port))
     ndpi_add_connection_as_bittorrent(ndpi_struct, flow, -1, 0, NDPI_CONFIDENCE_DPI_CACHE);
   else
     NDPI_EXCLUDE_PROTO(ndpi_struct, flow);
diff --git a/src/lib/protocols/ftp_data.c b/src/lib/protocols/ftp_data.c
index 4a3ad973d..2c3f06d97 100644
--- a/src/lib/protocols/ftp_data.c
+++ b/src/lib/protocols/ftp_data.c
@@ -232,7 +232,7 @@ static void ndpi_check_ftp_data(struct ndpi_detection_module_struct *ndpi_struct
     Make sure we see the beginning of the connection as otherwise we might have
     false positive results
   */
-  if(flow->l4.tcp.seen_syn) {
+  if(ndpi_seen_flow_beginning(flow)) {
     if((packet->payload_packet_len > 0)
        && (ndpi_match_file_header(ndpi_struct, flow)
 	   || ndpi_match_ftp_data_directory(ndpi_struct, flow) 
diff --git a/src/lib/protocols/icecast.c b/src/lib/protocols/icecast.c
index ce8b20c12..c8dac6b48 100644
--- a/src/lib/protocols/icecast.c
+++ b/src/lib/protocols/icecast.c
@@ -60,14 +60,12 @@ void ndpi_search_icecast_tcp(struct ndpi_detection_module_struct *ndpi_struct, s
     }
   }
 
-  if(flow == NULL) return;
-    
-  if((packet->packet_direction == flow->setup_packet_direction)
+  if(ndpi_current_pkt_from_client_to_server(packet, flow)
       && (flow->packet_counter < 10)) {
     return;
   }
 
-  if(packet->packet_direction != flow->setup_packet_direction) {
+  if(ndpi_current_pkt_from_server_to_client(packet, flow)) {
     /* server answer, now test Server for Icecast */
 
     ndpi_parse_packet_line_info(ndpi_struct, flow);
diff --git a/src/lib/protocols/lotus_notes.c b/src/lib/protocols/lotus_notes.c
index ff5c9cf71..376507f23 100644
--- a/src/lib/protocols/lotus_notes.c
+++ b/src/lib/protocols/lotus_notes.c
@@ -37,11 +37,8 @@ static void ndpi_check_lotus_notes(struct ndpi_detection_module_struct *ndpi_str
 
   flow->l4.tcp.lotus_notes_packet_id++;
   
-  if((flow->l4.tcp.lotus_notes_packet_id == 1)
-     /* We have seen the 3-way handshake */
-     && flow->l4.tcp.seen_syn
-     && flow->l4.tcp.seen_syn_ack
-     && flow->l4.tcp.seen_ack) {
+  if((flow->l4.tcp.lotus_notes_packet_id == 1) &&
+     ndpi_seen_flow_beginning(flow)) {
     if(payload_len > 16) {
       char lotus_notes_header[] = { 0x00, 0x00, 0x02, 0x00, 0x00, 0x40, 0x02, 0x0F };