Flow: keep track of "dissectors" (#2828)

In the flow, we should keep track of state of "dissectors", not
"protocols". This way, flow structure doesn't depend anymore on
the max number of protocols.
This is also the first step into fixing #2136

5 files changed, 75 insertions, 68 deletions

diff --git a/src/include/ndpi_api.h b/src/include/ndpi_api.h
index 60d0ea3e3..c06319aef 100644
--- a/src/include/ndpi_api.h
+++ b/src/include/ndpi_api.h
@@ -273,29 +273,6 @@ extern "C" {
   void ndpi_exit_detection_module(struct ndpi_detection_module_struct *ndpi_struct);
 
   /**
-   * Sets a single protocol bitmask
-   * This function does not increment the index of the callback_buffer
-   *
-   * @par label                    = string for the protocol name
-   * @par ndpi_struct              = the detection module
-   * @par idx                      = the index of the callback_buffer
-   * @par func                     = function pointer of the protocol search
-   * @par ndpi_selection_bitmask   = the protocol selected bitmask
-   * @par b_save_bitmask_unknow    = if set as "true" save the detection bitmask as unknow
-   * @par b_add_detection_bitmask  = if set as "true" add the protocol bitmask to the detection bitmask
-   *
-   */
-  void ndpi_set_bitmask_protocol_detection(char *label,
-					   struct ndpi_detection_module_struct *ndpi_struct,
-					   const u_int32_t idx,
-					   u_int16_t ndpi_protocol_id,
-					   void (*func) (struct ndpi_detection_module_struct *,
-							 struct ndpi_flow_struct *flow),
-					   const NDPI_SELECTION_BITMASK_PROTOCOL_SIZE ndpi_selection_bitmask,
-					   u_int8_t b_save_bitmask_unknow,
-					   u_int8_t b_add_detection_bitmask);
-
-  /**
    * Sets the protocol bitmask2
    *
    * @par ndpi_struct        = the detection module
diff --git a/src/include/ndpi_define.h.in b/src/include/ndpi_define.h.in
index 66ef9a5d4..be439cbc3 100644
--- a/src/include/ndpi_define.h.in
+++ b/src/include/ndpi_define.h.in
@@ -139,6 +139,10 @@
 #define NDPI_ISSET_BIT(num, n)  (num & (1ULL << ( n )))
 #define NDPI_ZERO_BIT(num)      num = 0
 
+#define NDPI_DISSECTOR_BITMASK                  ndpi_dissector_bitmask_struct_t
+#define NDPI_DISSECTOR_BITMASK_IS_SET(p, n)     NDPI_ISSET(&(p), (n))
+#define NDPI_DISSECTOR_BITMASK_SET(p, n)        NDPI_SET(&(p), (n) & NDPI_NUM_BITS_MASK)
+
 /* this is a very very tricky macro *g*,
  * the compiler will remove all shifts here if the protocol is static...
  */
diff --git a/src/include/ndpi_private.h b/src/include/ndpi_private.h
index a1d8179d1..df14bd8e9 100644
--- a/src/include/ndpi_private.h
+++ b/src/include/ndpi_private.h
@@ -58,7 +58,6 @@ struct call_function_struct {
   void (*func) (struct ndpi_detection_module_struct *, struct ndpi_flow_struct *flow);
   NDPI_SELECTION_BITMASK_PROTOCOL_SIZE ndpi_selection_bitmask;
   u_int16_t ndpi_protocol_id;
-  u_int8_t detection_feature;
 };
 
 struct subprotocol_conf_struct {
@@ -608,6 +607,16 @@ struct ndpi_detection_module_struct {
 
 /* Generic */
 
+void ndpi_set_bitmask_protocol_detection(char *label,
+                                         struct ndpi_detection_module_struct *ndpi_struct,
+                                         const u_int32_t idx,
+                                         u_int16_t ndpi_protocol_id,
+                                         void (*func) (struct ndpi_detection_module_struct *,
+                                                       struct ndpi_flow_struct *flow),
+                                         const NDPI_SELECTION_BITMASK_PROTOCOL_SIZE ndpi_selection_bitmask,
+                                         u_int8_t b_save_bitmask_unknow,
+                                         u_int8_t b_add_detection_bitmask);
+
 char *strptime(const char *s, const char *format, struct tm *tm);
 
 u_int8_t iph_is_valid_and_not_fragmented(const struct ndpi_iphdr *iph, const u_int16_t ipsize);
diff --git a/src/include/ndpi_typedefs.h b/src/include/ndpi_typedefs.h
index 3d40a15a9..6abe3b713 100644
--- a/src/include/ndpi_typedefs.h
+++ b/src/include/ndpi_typedefs.h
@@ -272,6 +272,18 @@ typedef struct ndpi_protocol_bitmask_struct {
   ndpi_ndpi_mask fds_bits[NDPI_NUM_FDS_BITS];
 } ndpi_protocol_bitmask_struct_t;
 
+
+#define NDPI_MAX_NUM_DISSECTORS         288      /* Multiple of 32, i.e. 8 * sizeof(ndpi_ndpi_mask) */
+#ifdef NDPI_CFFI_PREPROCESSING
+#undef NDPI_NUM_FDS_BITS
+#define NDPI_NUM_FDS_BITS_DISSECTORS    9
+#else
+#define NDPI_NUM_FDS_BITS_DISSECTORS    NDPI_MAX_NUM_DISSECTORS / (8 * sizeof(ndpi_ndpi_mask))
+#endif
+typedef struct ndpi_dissector_bitmask_struct {
+  ndpi_ndpi_mask fds_bits[NDPI_NUM_FDS_BITS_DISSECTORS];
+} ndpi_dissector_bitmask_struct_t;
+
 struct ndpi_detection_module_struct;
 
 /* NDPI_DEBUG_FUNCTION_PTR (cast) */
@@ -1191,7 +1203,7 @@ typedef struct ndpi_proto_defaults {
   u_int8_t isClearTextProto:1, isAppProtocol:1, _notused:6;
   u_int16_t *subprotocols;
   u_int32_t subprotocol_count;
-  u_int16_t protoId, protoIdx;
+  u_int16_t protoId, dissector_idx;
   u_int16_t tcp_default_ports[MAX_DEFAULT_PORTS], udp_default_ports[MAX_DEFAULT_PORTS];
   ndpi_protocol_breed_t protoBreed;
   ndpi_protocol_qoe_category_t qoeCategory;
@@ -1630,8 +1642,7 @@ struct ndpi_flow_struct {
   /* **Packet** metadata for flows where monitoring is enabled. It is reset after each packet! */
   struct ndpi_metadata_monitoring *monit;
 
-  /* protocols which have marked a connection as this connection cannot be protocol XXX, multiple u_int64_t */
-  NDPI_PROTOCOL_BITMASK excluded_protocol_bitmask;
+  NDPI_DISSECTOR_BITMASK excluded_dissectors_bitmask;
 
   /* NDPI_PROTOCOL_BITTORRENT */
   u_int8_t bittorrent_stage;		      // can be 0 - 255
diff --git a/src/lib/ndpi_main.c b/src/lib/ndpi_main.c
index 6f1cc2909..84083b243 100644
--- a/src/lib/ndpi_main.c
+++ b/src/lib/ndpi_main.c
@@ -526,7 +526,7 @@ void ndpi_exclude_protocol(struct ndpi_detection_module_struct *ndpi_str, struct
     (void)_func;
     (void)_line;
 #endif
-    NDPI_ADD_PROTOCOL_TO_BITMASK(flow->excluded_protocol_bitmask, protocol_id);
+    NDPI_DISSECTOR_BITMASK_SET(flow->excluded_dissectors_bitmask, ndpi_str->proto_defaults[protocol_id].dissector_idx);
   }
 }
 
@@ -5733,7 +5733,6 @@ int load_protocols_file_fd(struct ndpi_detection_module_struct *ndpi_str, FILE *
 
 /* ******************************************************************** */
 
-/* ntop */
 void ndpi_set_bitmask_protocol_detection(char *label, struct ndpi_detection_module_struct *ndpi_str,
                                          const u_int32_t idx,
                                          u_int16_t ndpi_protocol_id,
@@ -5742,9 +5741,20 @@ void ndpi_set_bitmask_protocol_detection(char *label, struct ndpi_detection_modu
                                          const NDPI_SELECTION_BITMASK_PROTOCOL_SIZE ndpi_selection_bitmask,
                                          u_int8_t b_save_bitmask_unknow, u_int8_t b_add_detection_bitmask) {
   (void)label;
-  /*
-    Compare specify protocol bitmask with main detection bitmask
-  */
+
+
+  if(idx >= NDPI_MAX_NUM_DISSECTORS) {
+    /*
+     * You need to increase NDPI_MAX_NUM_DISSECTORS define and recompile everything!
+     * Please note that custom protocols are independent from NDPI_MAX_NUM_DISSECTORS, so
+     * if you hit this error is because you are already changing the code
+     * (adding a new dissector)...
+     */
+    NDPI_LOG_ERR(ndpi_str, "[NDPI] Internal Error. Too many dissectors!!\n");
+    /* Not sure what to do here...*/
+    return;
+  }
+
   if(is_proto_enabled(ndpi_str, ndpi_protocol_id)) {
 #ifdef NDPI_ENABLE_DEBUG_MESSAGES
     NDPI_LOG_DBG2(ndpi_str,
@@ -5753,7 +5763,7 @@ void ndpi_set_bitmask_protocol_detection(char *label, struct ndpi_detection_modu
 		  label, idx, ndpi_protocol_id);
 #endif
 
-    if(ndpi_str->proto_defaults[ndpi_protocol_id].protoIdx != 0) {
+    if(ndpi_str->proto_defaults[ndpi_protocol_id].dissector_idx != 0) {
       NDPI_LOG_DBG2(ndpi_str, "[NDPI] Internal error: protocol %s/%u has been already registered\n", label,
 		    ndpi_protocol_id);
 #ifdef NDPI_ENABLE_DEBUG_MESSAGES
@@ -5766,7 +5776,7 @@ void ndpi_set_bitmask_protocol_detection(char *label, struct ndpi_detection_modu
       Set function and index protocol within proto_default structure for port protocol detection
       and callback_buffer function for DPI protocol detection
     */
-    ndpi_str->proto_defaults[ndpi_protocol_id].protoIdx = idx;
+    ndpi_str->proto_defaults[ndpi_protocol_id].dissector_idx = idx;
     ndpi_str->proto_defaults[ndpi_protocol_id].func = ndpi_str->callback_buffer[idx].func = func;
     ndpi_str->callback_buffer[idx].ndpi_protocol_id = ndpi_protocol_id;
 
@@ -6588,8 +6598,12 @@ static int ndpi_callback_init(struct ndpi_detection_module_struct *ndpi_str) {
 
   ndpi_enabled_callbacks_init(ndpi_str,detection_bitmask,0);
 
-  /*   When the module ends, it is necessary to free the memory ndpi_str->callback_buffer and
-       ndpi_str->callback_buffer_tcp_payload  */
+  NDPI_LOG_DBG(ndpi_str, "Tot num dissectors: %d (TCP: %d, TCP_NO_PAYLOAD: %d, UDP: %d, NO_TCP_UDP: %d\n",
+               ndpi_str->callback_buffer_size,
+               ndpi_str->callback_buffer_size_tcp_payload,
+               ndpi_str->callback_buffer_size_tcp_no_payload,
+               ndpi_str->callback_buffer_size_udp,
+               ndpi_str->callback_buffer_size_non_tcp_udp);
 
   return 0;
 }
@@ -7666,12 +7680,11 @@ static u_int32_t check_ndpi_subprotocols(struct ndpi_detection_module_struct * c
       continue;
     }
 
-    u_int16_t subproto_index = ndpi_str->proto_defaults[subproto_id].protoIdx;
+    u_int16_t subproto_index = ndpi_str->proto_defaults[subproto_id].dissector_idx;
 
     if((ndpi_str->callback_buffer[subproto_index].ndpi_selection_bitmask & ndpi_selection_packet) ==
        ndpi_str->callback_buffer[subproto_index].ndpi_selection_bitmask &&
-       NDPI_BITMASK_COMPARE(flow->excluded_protocol_bitmask,
-			    ndpi_str->callback_buffer[subproto_index].excluded_protocol_bitmask) == 0 &&
+       !NDPI_DISSECTOR_BITMASK_IS_SET(flow->excluded_dissectors_bitmask, subproto_index) &&
        NDPI_BITMASK_COMPARE(ndpi_str->callback_buffer[subproto_index].detection_bitmask,
 			    detection_bitmask) != 0) {
       ndpi_str->callback_buffer[subproto_index].func(ndpi_str, flow);
@@ -7688,49 +7701,42 @@ static u_int32_t check_ndpi_detection_func(struct ndpi_detection_module_struct *
 					   struct ndpi_flow_struct * const flow,
 					   NDPI_SELECTION_BITMASK_PROTOCOL_SIZE const ndpi_selection_packet,
 					   struct call_function_struct const * const callback_buffer,
-					   uint32_t callback_buffer_size,
-					   int is_tcp_without_payload) {
+					   uint32_t callback_buffer_size) {
   void *func = NULL;
   u_int32_t num_calls = 0;
   /* First callback is associated to classification by-port,
      if we don't already have a partial classification */
   u_int16_t fast_callback_protocol_id = flow->fast_callback_protocol_id ? flow->fast_callback_protocol_id : flow->guessed_protocol_id;
-  u_int16_t proto_index = ndpi_str->proto_defaults[fast_callback_protocol_id].protoIdx;
-  u_int16_t proto_id = ndpi_str->proto_defaults[fast_callback_protocol_id].protoId;
+  u_int16_t dissector_idx = ndpi_str->proto_defaults[fast_callback_protocol_id].dissector_idx;
+  u_int16_t proto_id;
   NDPI_PROTOCOL_BITMASK detection_bitmask;
   u_int32_t a;
 
   NDPI_SAVE_AS_BITMASK(detection_bitmask, flow->detected_protocol_stack[0]);
 
-  if((proto_id != NDPI_PROTOCOL_UNKNOWN) &&
-     NDPI_BITMASK_COMPARE(flow->excluded_protocol_bitmask,
-			  ndpi_str->callback_buffer[proto_index].excluded_protocol_bitmask) == 0 &&
-     NDPI_BITMASK_COMPARE(ndpi_str->callback_buffer[proto_index].detection_bitmask, detection_bitmask) != 0 &&
-     (ndpi_str->callback_buffer[proto_index].ndpi_selection_bitmask & ndpi_selection_packet) ==
-     ndpi_str->callback_buffer[proto_index].ndpi_selection_bitmask)
-    {
-      if((fast_callback_protocol_id != NDPI_PROTOCOL_UNKNOWN) &&
-	 (ndpi_str->proto_defaults[fast_callback_protocol_id].func != NULL) &&
-	 (is_tcp_without_payload == 0 ||
-	  ((ndpi_str->callback_buffer[proto_index].ndpi_selection_bitmask &
-	    NDPI_SELECTION_BITMASK_PROTOCOL_HAS_PAYLOAD) == 0)))
-	{
-	  ndpi_str->proto_defaults[fast_callback_protocol_id].func(ndpi_str, flow);
-	  func = ndpi_str->proto_defaults[fast_callback_protocol_id].func;
-	  num_calls++;
-	}
-    }
+  if(fast_callback_protocol_id != NDPI_PROTOCOL_UNKNOWN &&
+     ndpi_str->proto_defaults[fast_callback_protocol_id].func &&
+     !NDPI_DISSECTOR_BITMASK_IS_SET(flow->excluded_dissectors_bitmask, dissector_idx) &&
+     (ndpi_str->callback_buffer[dissector_idx].ndpi_selection_bitmask & ndpi_selection_packet) ==
+     ndpi_str->callback_buffer[dissector_idx].ndpi_selection_bitmask &&
+     NDPI_BITMASK_COMPARE(ndpi_str->callback_buffer[dissector_idx].detection_bitmask, detection_bitmask) != 0) {
+    ndpi_str->proto_defaults[fast_callback_protocol_id].func(ndpi_str, flow);
+    func = ndpi_str->proto_defaults[fast_callback_protocol_id].func;
+    num_calls++;
+  }
 
   if(flow->detected_protocol_stack[0] == NDPI_PROTOCOL_UNKNOWN)
     {
       /* TODO: optimize as today we're doing a linear scan */
 
       for (a = 0; a < callback_buffer_size; a++) {
+        proto_id = callback_buffer[a].ndpi_protocol_id;
+        dissector_idx = ndpi_str->proto_defaults[proto_id].dissector_idx;
+
         if((func != callback_buffer[a].func) &&
 	   (callback_buffer[a].ndpi_selection_bitmask & ndpi_selection_packet) ==
 	   callback_buffer[a].ndpi_selection_bitmask &&
-	   NDPI_BITMASK_COMPARE(flow->excluded_protocol_bitmask,
-				callback_buffer[a].excluded_protocol_bitmask) == 0 &&
+	   !NDPI_DISSECTOR_BITMASK_IS_SET(flow->excluded_dissectors_bitmask, dissector_idx) &&
 	   NDPI_BITMASK_COMPARE(callback_buffer[a].detection_bitmask,
 				detection_bitmask) != 0)
 	  {
@@ -7761,7 +7767,7 @@ u_int32_t check_ndpi_other_flow_func(struct ndpi_detection_module_struct *ndpi_s
 {
   return check_ndpi_detection_func(ndpi_str, flow, *ndpi_selection_packet,
 				   ndpi_str->callback_buffer_non_tcp_udp,
-				   ndpi_str->callback_buffer_size_non_tcp_udp, 0);
+				   ndpi_str->callback_buffer_size_non_tcp_udp);
 }
 
 /* ************************************************ */
@@ -7771,7 +7777,7 @@ static u_int32_t check_ndpi_udp_flow_func(struct ndpi_detection_module_struct *n
 					  NDPI_SELECTION_BITMASK_PROTOCOL_SIZE *ndpi_selection_packet) {
   return check_ndpi_detection_func(ndpi_str, flow, *ndpi_selection_packet,
 				   ndpi_str->callback_buffer_udp,
-				   ndpi_str->callback_buffer_size_udp, 0);
+				   ndpi_str->callback_buffer_size_udp);
 }
 
 /* ************************************************ */
@@ -7783,12 +7789,12 @@ static u_int32_t check_ndpi_tcp_flow_func(struct ndpi_detection_module_struct *n
   if(ndpi_str->packet.payload_packet_len != 0) {
     return check_ndpi_detection_func(ndpi_str, flow, *ndpi_selection_packet,
 				     ndpi_str->callback_buffer_tcp_payload,
-				     ndpi_str->callback_buffer_size_tcp_payload, 0);
+				     ndpi_str->callback_buffer_size_tcp_payload);
   } else {
     /* no payload */
     return check_ndpi_detection_func(ndpi_str, flow, *ndpi_selection_packet,
 				     ndpi_str->callback_buffer_tcp_no_payload,
-				     ndpi_str->callback_buffer_size_tcp_no_payload, 1);
+				     ndpi_str->callback_buffer_size_tcp_no_payload);
   }
 }
 
@@ -10930,7 +10936,7 @@ const char *ndpi_get_l4_proto_name(ndpi_l4_proto_info proto) {
 ndpi_l4_proto_info ndpi_get_l4_proto_info(struct ndpi_detection_module_struct *ndpi_struct,
 					  u_int16_t ndpi_proto_id) {
   if(ndpi_struct && ndpi_proto_id < ndpi_struct->ndpi_num_supported_protocols) {
-    u_int16_t idx = ndpi_struct->proto_defaults[ndpi_proto_id].protoIdx;
+    u_int16_t idx = ndpi_struct->proto_defaults[ndpi_proto_id].dissector_idx;
     NDPI_SELECTION_BITMASK_PROTOCOL_SIZE bm = ndpi_struct->callback_buffer[idx].ndpi_selection_bitmask;
 
     if(bm & NDPI_SELECTION_BITMASK_PROTOCOL_INT_TCP)