Let the library returning the packet direction calculated internally (#2572)

wireshark, lua: add basic analysis of possible obfuscated flows

9 files changed, 122 insertions, 18 deletions

diff --git a/example/ndpiReader.c b/example/ndpiReader.c
index 32b7399cd..6fe6e3e00 100644
--- a/example/ndpiReader.c
+++ b/example/ndpiReader.c
@@ -236,6 +236,7 @@ struct ndpi_packet_trailer {
   u_int32_t magic; /* WIRESHARK_NTOP_MAGIC */
   ndpi_master_app_protocol proto;
   char name[16];
+  u_int8_t flags;
   ndpi_risk flow_risk;
   u_int16_t flow_score;
   u_int16_t flow_risk_info_len;
@@ -248,7 +249,7 @@ struct ndpi_packet_trailer {
 
 static pcap_dumper_t *extcap_dumper = NULL;
 static pcap_t *extcap_fifo_h = NULL;
-static char extcap_buf[16384];
+static char extcap_buf[65536 + sizeof(struct ndpi_packet_trailer)];
 static char *extcap_capture_fifo    = NULL;
 static u_int16_t extcap_packet_filter = (u_int16_t)-1;
 static int do_extcap_capture = 0;
@@ -4565,13 +4566,19 @@ static void ndpi_process_packet(u_char *args,
     memcpy(extcap_buf, packet, h.caplen);
     memset(trailer, 0, sizeof(struct ndpi_packet_trailer));
     trailer->magic = htonl(WIRESHARK_NTOP_MAGIC);
+    if(flow) {
+      trailer->flags = flow->current_pkt_from_client_to_server;
+      trailer->flags |= (flow->detection_completed << 2);
+    } else {
+      trailer->flags = 0 | (2 << 2);
+    }
     trailer->flow_risk = htonl64(flow_risk);
     trailer->flow_score = htons(ndpi_risk2score(flow_risk, &cli_score, &srv_score));
     trailer->flow_risk_info_len = ntohs(WIRESHARK_FLOW_RISK_INFO_SIZE);
-    if(flow->risk_str) {
+    if(flow && flow->risk_str) {
       strncpy(trailer->flow_risk_info, flow->risk_str, sizeof(trailer->flow_risk_info));
-      trailer->flow_risk_info[sizeof(trailer->flow_risk_info) - 1] = '\0';
     }
+    trailer->flow_risk_info[sizeof(trailer->flow_risk_info) - 1] = '\0';
     trailer->proto.master_protocol = htons(p.proto.master_protocol), trailer->proto.app_protocol = htons(p.proto.app_protocol);
     ndpi_protocol2name(ndpi_thread_info[thread_id].workflow->ndpi_struct, p, trailer->name, sizeof(trailer->name));
 
diff --git a/example/reader_util.c b/example/reader_util.c
index 287f133cd..c0c723da0 100644
--- a/example/reader_util.c
+++ b/example/reader_util.c
@@ -1733,7 +1733,6 @@ static struct ndpi_proto packet_processing(struct ndpi_workflow * workflow,
     flow->detected_protocol = ndpi_detection_process_packet(workflow->ndpi_struct, ndpi_flow,
 							    iph ? (uint8_t *)iph : (uint8_t *)iph6,
 							    ipsize, time_ms, &input_info);
-
     enough_packets |= ndpi_flow->fail_with_unknown;
     if(enough_packets || (flow->detected_protocol.proto.app_protocol != NDPI_PROTOCOL_UNKNOWN)) {
       if((!enough_packets)
@@ -1754,7 +1753,12 @@ static struct ndpi_proto packet_processing(struct ndpi_workflow * workflow,
 	process_ndpi_collected_info(workflow, flow);
       }
     }
+    /* Let's try to save client-server direction */
+    flow->current_pkt_from_client_to_server = input_info.in_pkt_dir;
+
     malloc_size_stats = 0;
+  } else {
+    flow->current_pkt_from_client_to_server = NDPI_IN_PKT_DIR_UNKNOWN; /* Unknown */
   }
 
 #if 0
diff --git a/example/reader_util.h b/example/reader_util.h
index 27b8e1c0b..ef15053cd 100644
--- a/example/reader_util.h
+++ b/example/reader_util.h
@@ -182,7 +182,7 @@ typedef struct ndpi_flow_info {
   struct ndpi_in6_addr dst_ip6; /* network order */
   u_int16_t src_port; /* network order */
   u_int16_t dst_port; /* network order */
-  u_int8_t detection_completed, protocol, bidirectional, check_extra_packets;
+  u_int8_t detection_completed, protocol, bidirectional, check_extra_packets, current_pkt_from_client_to_server;
   u_int16_t vlan_id;
   ndpi_packet_tunnel tunnel_type;
   struct ndpi_flow_struct *ndpi_flow;
diff --git a/python/ndpi/ndpi_build.py b/python/ndpi/ndpi_build.py
index 07aac0de3..e2eac98e4 100644
--- a/python/ndpi/ndpi_build.py
+++ b/python/ndpi/ndpi_build.py
@@ -57,7 +57,7 @@ ndpi_protocol ndpi_detection_process_packet(struct ndpi_detection_module_struct
                                             const unsigned char *packet,
                                             const unsigned short packetlen,
                                             const u_int64_t packet_time_ms,
-                                            const struct ndpi_flow_input_info *input_info);
+                                            struct ndpi_flow_input_info *input_info);
 ndpi_protocol ndpi_detection_giveup(struct ndpi_detection_module_struct *ndpi_struct,
                                     struct ndpi_flow_struct *flow,
                                     u_int8_t *protocol_was_guessed);
diff --git a/src/include/ndpi_api.h b/src/include/ndpi_api.h
index d2ba9816e..edfb497d4 100644
--- a/src/include/ndpi_api.h
+++ b/src/include/ndpi_api.h
@@ -334,7 +334,7 @@ extern "C" {
 				 const unsigned char *packet,
 				 const unsigned short packetlen,
 				 const u_int64_t packet_time_ms,
-				 const struct ndpi_flow_input_info *input_info);
+				 struct ndpi_flow_input_info *input_info);
 
   /**
    * Processes one packet and returns the ID of the detected protocol.
@@ -354,7 +354,7 @@ extern "C" {
 					      const unsigned char *packet,
 					      const unsigned short packetlen,
 					      const u_int64_t packet_time_ms,
-					      const struct ndpi_flow_input_info *input_info);
+					      struct ndpi_flow_input_info *input_info);
   /**
    * Get the main protocol of the passed flows for the detected module
    *
diff --git a/src/include/ndpi_private.h b/src/include/ndpi_private.h
index 3aa17ed3c..c7eef1e0b 100644
--- a/src/include/ndpi_private.h
+++ b/src/include/ndpi_private.h
@@ -404,7 +404,7 @@ struct ndpi_detection_module_struct {
 
   /* Current packet */
   struct ndpi_packet_struct packet;
-  const struct ndpi_flow_input_info *input_info;
+  struct ndpi_flow_input_info *input_info;
 
 #ifdef HAVE_NBPF
   u_int8_t num_nbpf_custom_proto;
diff --git a/src/include/ndpi_typedefs.h b/src/include/ndpi_typedefs.h
index fc0a2cf16..1973bd981 100644
--- a/src/include/ndpi_typedefs.h
+++ b/src/include/ndpi_typedefs.h
@@ -645,7 +645,7 @@ struct ndpi_gre_basehdr {
  * Optional information about flow management (per packet)
  */
 struct ndpi_flow_input_info {
-  unsigned char in_pkt_dir;
+  unsigned char in_pkt_dir; /* If unknown, the library might *returns* to the application the direction calculated internally */
   unsigned char seen_flow_beginning;
 };
 
diff --git a/src/lib/ndpi_main.c b/src/lib/ndpi_main.c
index 3e16ca5c1..ef5bab840 100644
--- a/src/lib/ndpi_main.c
+++ b/src/lib/ndpi_main.c
@@ -6804,7 +6804,7 @@ static int ndpi_init_packet(struct ndpi_detection_module_struct *ndpi_str,
 			    const u_int64_t current_time_ms,
 			    const unsigned char *packet_data,
 			    unsigned short packetlen,
-			    const struct ndpi_flow_input_info *input_info) {
+			    struct ndpi_flow_input_info *input_info) {
   struct ndpi_packet_struct *packet = &ndpi_str->packet;
   const struct ndpi_iphdr *decaps_iph = NULL;
   u_int16_t l3len;
@@ -7261,6 +7261,14 @@ static void ndpi_connection_tracking(struct ndpi_detection_module_struct *ndpi_s
 	ndpi_unset_risk(flow, NDPI_UNIDIRECTIONAL_TRAFFIC); /* Clear bit */
       }
     }
+
+    if(ndpi_str->input_info &&
+       ndpi_str->input_info->in_pkt_dir == NDPI_IN_PKT_DIR_UNKNOWN) {
+      if(current_pkt_from_client_to_server(ndpi_str, flow))
+        ndpi_str->input_info->in_pkt_dir = NDPI_IN_PKT_DIR_C_TO_S;
+      else
+        ndpi_str->input_info->in_pkt_dir = NDPI_IN_PKT_DIR_S_TO_C;
+    }
 }
 
 /* ************************************************ */
@@ -7959,7 +7967,7 @@ void ndpi_process_extra_packet(struct ndpi_detection_module_struct *ndpi_str,
 			       struct ndpi_flow_struct *flow,
 			       const unsigned char *packet_data, const unsigned short packetlen,
 			       const u_int64_t current_time_ms,
-			       const struct ndpi_flow_input_info *input_info) {
+			       struct ndpi_flow_input_info *input_info) {
   if(flow == NULL)
     return;
 
@@ -8562,7 +8570,7 @@ static ndpi_protocol ndpi_internal_detection_process_packet(struct ndpi_detectio
 							    const unsigned char *packet_data,
 							    const unsigned short packetlen,
 							    const u_int64_t current_time_ms,
-							    const struct ndpi_flow_input_info *input_info) {
+							    struct ndpi_flow_input_info *input_info) {
   struct ndpi_packet_struct *packet;
   NDPI_SELECTION_BITMASK_PROTOCOL_SIZE ndpi_selection_packet;
   u_int32_t num_calls = 0;
@@ -8593,6 +8601,10 @@ static ndpi_protocol ndpi_internal_detection_process_packet(struct ndpi_detectio
   if(ndpi_str->cfg.max_packets_to_process > 0 && flow->num_processed_pkts >= ndpi_str->cfg.max_packets_to_process) {
     flow->extra_packets_func = NULL; /* To allow ndpi_extra_dissection_possible() to fail */
     flow->fail_with_unknown = 1;
+    /* Let's try to update ndpi_str->input_info->in_pkt_dir even in this case.
+     * It is quite uncommon, so we are not going to spend a lot of resources here... */
+    if(ndpi_init_packet(ndpi_str, flow, current_time_ms, packet_data, packetlen, input_info) == 0)
+      ndpi_connection_tracking(ndpi_str, flow);
     return(ret); /* Avoid spending too much time with this flow */
   }
 
@@ -8892,7 +8904,7 @@ static ndpi_protocol ndpi_internal_detection_process_packet(struct ndpi_detectio
 ndpi_protocol ndpi_detection_process_packet(struct ndpi_detection_module_struct *ndpi_str,
 					    struct ndpi_flow_struct *flow, const unsigned char *packet_data,
 					    const unsigned short packetlen, const u_int64_t current_time_ms,
-					    const struct ndpi_flow_input_info *input_info) {
+					    struct ndpi_flow_input_info *input_info) {
   ndpi_protocol p  = ndpi_internal_detection_process_packet(ndpi_str, flow, packet_data,
 							    packetlen, current_time_ms,
 							    input_info);
diff --git a/wireshark/ndpi.lua b/wireshark/ndpi.lua
index adc0fd786..cfd41ab05 100644
--- a/wireshark/ndpi.lua
+++ b/wireshark/ndpi.lua
@@ -35,6 +35,19 @@ ndpi_fds.magic                = ProtoField.new("nDPI Magic", "ndpi.magic", ftype
 ndpi_fds.network_protocol     = ProtoField.new("nDPI Network Protocol", "ndpi.protocol.network", ftypes.UINT8, nil, base.DEC)
 ndpi_fds.application_protocol = ProtoField.new("nDPI Application Protocol", "ndpi.protocol.application", ftypes.UINT8, nil, base.DEC)
 ndpi_fds.name                 = ProtoField.new("nDPI Protocol Name", "ndpi.protocol.name", ftypes.STRING)
+ndpi_fds.flags                = ProtoField.new("nDPI Flags", "ndpi.flags", ftypes.UINT8, nil, base.HEX)
+local dir_types = {
+        [0] = "Unknown Direction",
+        [1] = "Client to Server Direction",
+        [2] = "Server to Client Direction",
+}
+ndpi_fds.flags_direction      = ProtoField.new("nDPI Direction", "ndpi.flags.direction", ftypes.UINT8, dir_types, base.DEC, 0x03)
+local dpi_state_types = {
+        [0] = "Inspecting",
+        [1] = "From Inspecting to Done",
+        [2] = "Done",
+}
+ndpi_fds.flags_dpi_state      = ProtoField.new("nDPI DPI state", "ndpi.flags.dpi_state", ftypes.UINT8, dpi_state_types, base.DEC, 0xC)
 ndpi_fds.flow_risk            = ProtoField.new("nDPI Flow Risk", "ndpi.flow_risk", ftypes.UINT64, nil, base.HEX)
 ndpi_fds.flow_score           = ProtoField.new("nDPI Flow Score", "ndpi.flow_score", ftypes.UINT32)
 ndpi_fds.flow_risk_info_len   = ProtoField.new("nDPI Flow Risk Info Length", "ndpi.flow_risk_info_len", ftypes.UINT16, nil, base.DEC)
@@ -54,8 +67,6 @@ ndpi_fds.metadata_length      = ProtoField.new("nDPI Metadata Length", "ndpi.met
 ndpi_fds.metadata_value       = ProtoField.new("nDPI Metadata Value", "ndpi.metadata.value", ftypes.BYTES)
 -- Specific fields
 ndpi_fds.metadata_server_name = ProtoField.new("nDPI Server Name", "ndpi.metadata.server_name", ftypes.STRING)
-ndpi_fds.metadata_ja3c        = ProtoField.new("nDPI JA3C", "ndpi.metadata.ja3c", ftypes.STRING)
-ndpi_fds.metadata_ja3s        = ProtoField.new("nDPI JA3S", "ndpi.metadata.ja3s", ftypes.STRING)
 ndpi_fds.metadata_ja4c        = ProtoField.new("nDPI JA4C", "ndpi.metadata.ja4c", ftypes.STRING)
 
 
@@ -193,6 +204,9 @@ local tot_tls_flows          = 0
 local http_ua                = {}
 local tot_http_ua_flows      = 0
 
+local possible_obfuscated_servers = {}
+local tot_obfuscated_flows        = 0
+
 local flows                  = {}
 local tot_flows              = 0
 
@@ -220,6 +234,8 @@ local debug                  = false
 
 local dump_timeseries = false
 
+local track_obfuscated_servers = true
+
 local dissect_ndpi_trailer = true
 
 local dump_file = "/tmp/wireshark-influx.txt"
@@ -440,6 +456,10 @@ function ndpi_proto.init()
    http_ua                = {}
    tot_http_ua_flows      = 0
 
+   -- Obfuscated servers
+   possible_obfuscated_servers = {}
+   tot_obfuscated_flows        = 0
+
    -- Flows
    flows                  = {}
    tot_flows              = 0
@@ -1111,6 +1131,13 @@ function ndpi_proto.dissector(tvb, pinfo, tree)
 	       --print(network_protocol .. "/" .. application_protocol .. "/".. name)
 	    end
 
+	    ndpi_subtree:add(ndpi_fds.flags, trailer_tvb(offset, 1))
+	    ndpi_subtree:add(ndpi_fds.flags_direction, trailer_tvb(offset, 1))
+	    local direction = trailer_tvb(offset, 1):bitfield(6,2) -- From left to right!! -> inverted values
+	    ndpi_subtree:add(ndpi_fds.flags_dpi_state, trailer_tvb(offset, 1))
+	    local dpi_state = trailer_tvb(offset, 1):bitfield(4,2) -- From left to right!! -> inverted values
+	    offset = offset + 1
+
 	    flow_risk_tree = ndpi_subtree:add(ndpi_fds.flow_risk, trailer_tvb(offset, 8))
 	    flow_risk = trailer_tvb(offset, 8):uint64() -- UInt64 object!
 	    offset = offset + 8
@@ -1133,11 +1160,15 @@ function ndpi_proto.dissector(tvb, pinfo, tree)
 	       
 	       for i=0,63 do
 		 if flow_risks[i] ~= nil then
-	            flow_risk_tree:add(flow_risks[i], trailer_tvb(24, 8))
+	            flow_risk_tree:add(flow_risks[i], trailer_tvb(25, 8))
 		 end
 
 	       end
-	       flow_risk_tree:add(flow_risks[64], trailer_tvb(24, 8)) -- Unused bits in flow risk bitmask
+	       flow_risk_tree:add(flow_risks[64], trailer_tvb(25, 8)) -- Unused bits in flow risk bitmask
+
+	       flow_risk_obfuscated_traffic = trailer_tvb(25, 8):bitfield(7,1) -- From left to right
+	    else
+	       flow_risk_obfuscated_traffic = 0
 	    end
 	    
 	    if(flow_score > 0) then
@@ -1226,6 +1257,29 @@ function ndpi_proto.dissector(tvb, pinfo, tree)
 
 	       ndpi_flows[flowkey] = ndpi_flows[flowkey] + pinfo.len
 	    end
+
+	    if(track_obfuscated_servers and pinfo.visited == false) then
+	      -- Only once per flow, when DPI ends
+	      if(dpi_state == 1) then
+		if(direction == 2) then -- current packet from server to client
+	          key = tostring(pinfo.src) .. ":" .. getstring(pinfo.src_port) .. " " .. name
+		else
+	          key = tostring(pinfo.dst) .. ":" .. getstring(pinfo.dst_port) .. " " .. name
+		end
+		if(possible_obfuscated_servers[key] == nil) then
+		  possible_obfuscated_servers[key] = {1, flow_risk_obfuscated_traffic}
+		else
+		  possible_obfuscated_servers[key][1] = possible_obfuscated_servers[key][1] + 1
+		  if(flow_risk_obfuscated_traffic == 1) then
+		    possible_obfuscated_servers[key][2] = possible_obfuscated_servers[key][2] + 1
+		  end
+		end
+
+		if(flow_risk_obfuscated_traffic == 1) then
+		  tot_obfuscated_flows = tot_obfuscated_flows + 1
+		end
+	      end
+	    end
 	 end
       end -- nDPI
 
@@ -1569,6 +1623,32 @@ end
 
 -- ###############################################
 
+local function obfuscated_servers_dialog_menu()
+   local win = TextWindow.new("Obfuscated Servers Analysis");
+   local label = ""
+   local tot = 0
+   local i
+
+   if(tot_obfuscated_flows > 0) then
+      i = 0
+      label = label .. "Server\t\tProtocol\tTotal Flows\tObfuscated flows\n"
+      for k,v in pairsByKeys(possible_obfuscated_servers, rev) do
+         for token in string.gmatch(k, "[^%s]+") do -- split key in two token (for beter formatting): ip:port and protocol
+	   label = label .. token .. "\t"
+         end
+	 label = label .. v[1] .. "\t\t" .. v[2] .. "\n"
+      end
+      label = label .. "\n\nTotal obfuscated flows: " .. tot_obfuscated_flows .. "\n"
+   else
+      label = "No possible Obfuscated Servers detected"
+   end
+
+   win:set(label)
+   win:add_button("Clear", function() win:clear() end)
+end
+
+-- ###############################################
+
 local function http_ua_dialog_menu()
    local win = TextWindow.new("HTTP User Agent");
    local label = ""
@@ -1766,6 +1846,7 @@ register_menu("ntop/TCP Analysis", tcp_dialog_menu, MENU_TOOLS_UNSORTED)
 register_menu("ntop/VLAN",         vlan_dialog_menu, MENU_TOOLS_UNSORTED)
 register_menu("ntop/Latency/Network",      rtt_dialog_menu, MENU_TOOLS_UNSORTED)
 register_menu("ntop/Latency/Application",  appl_rtt_dialog_menu, MENU_TOOLS_UNSORTED)
+register_menu("ntop/Obfuscated Servers Analysis", obfuscated_servers_dialog_menu, MENU_TOOLS_UNSORTED)
 
 -- ###############################################