wireshark: lua: export some metadata

Export some metadata (for the moment, SNI and TLS fingerprints) to
Wireshark/tshark via extcap.
Note that:
* metadata are exported only once per flow
* metadata are exported (all together) when nDPI stopped processing
the flow

Still room for a lot of improvements!
In particular:
* we need to add some boundary checks (if we are going to export other
attributes)
* we should try to have a variable length trailer

6 files changed, 135 insertions, 9 deletions

diff --git a/example/ndpiReader.c b/example/ndpiReader.c
index ed6ed79e8..fca7e3e94 100644
--- a/example/ndpiReader.c
+++ b/example/ndpiReader.c
@@ -216,6 +216,18 @@ struct receiver {
 struct receiver *receivers = NULL, *topReceivers = NULL;
 
 #define WIRESHARK_NTOP_MAGIC 0x19680924
+#define WIRESHARK_METADATA_SIZE	256
+
+#define WIRESHARK_METADATA_SERVERNAME	0x01
+#define WIRESHARK_METADATA_JA3C		0x02
+#define WIRESHARK_METADATA_JA3S		0x03
+#define WIRESHARK_METADATA_JA4C		0x04
+
+struct ndpi_packet_tlv {
+  u_int16_t type;
+  u_int16_t length;
+  unsigned char data[];
+};
 
 PACK_ON
 struct ndpi_packet_trailer {
@@ -224,6 +236,9 @@ struct ndpi_packet_trailer {
   ndpi_risk flow_risk;
   u_int16_t flow_score;
   char name[16];
+  /* TLV of attributes. Having a max and fixed size for all the metadata
+     is not efficient but greatly improves detection of the trailer by Wireshark */
+  unsigned char metadata[WIRESHARK_METADATA_SIZE];
 } PACK_OFF;
 
 static pcap_dumper_t *extcap_dumper = NULL;
@@ -4424,6 +4439,7 @@ static void ndpi_process_packet(u_char *args,
 				const u_char *packet) {
   struct ndpi_proto p;
   ndpi_risk flow_risk;
+  struct ndpi_flow_info *flow;
   u_int16_t thread_id = *((u_int16_t*)args);
 
   /* allocate an exact size buffer to check overflows */
@@ -4434,7 +4450,7 @@ static void ndpi_process_packet(u_char *args,
   }
 
   memcpy(packet_checked, packet, header->caplen);
-  p = ndpi_workflow_process_packet(ndpi_thread_info[thread_id].workflow, header, packet_checked, &flow_risk);
+  p = ndpi_workflow_process_packet(ndpi_thread_info[thread_id].workflow, header, packet_checked, &flow_risk, &flow);
 
   if(!pcap_start.tv_sec) pcap_start.tv_sec = header->ts.tv_sec, pcap_start.tv_usec = header->ts.tv_usec;
   pcap_end.tv_sec = header->ts.tv_sec, pcap_end.tv_usec = header->ts.tv_usec;
@@ -4495,6 +4511,57 @@ static void ndpi_process_packet(u_char *args,
     trailer->flow_score = htons(ndpi_risk2score(flow_risk, &cli_score, &srv_score));
     trailer->master_protocol = htons(p.master_protocol), trailer->app_protocol = htons(p.app_protocol);
     ndpi_protocol2name(ndpi_thread_info[thread_id].workflow->ndpi_struct, p, trailer->name, sizeof(trailer->name));
+
+    /* Metadata */
+    /* Metadata are (all) available in `flow` only after nDPI completed its work!
+       We export them only once */
+    /* TODO: boundary check. Right now there is always enough room, but we should check it if we are
+       going to extend the list of the metadata exported */
+    struct ndpi_packet_tlv *tlv = (struct ndpi_packet_tlv *)trailer->metadata;
+    int tot_len = 0;
+    if(flow && flow->detection_completed == 1) {
+      if(flow->host_server_name[0] != '\0') {
+        tlv->type = ntohs(WIRESHARK_METADATA_SERVERNAME);
+        tlv->length = ntohs(sizeof(flow->host_server_name));
+        memcpy(tlv->data, flow->host_server_name, sizeof(flow->host_server_name));
+        /* TODO: boundary check */
+        tot_len += 4 + htons(tlv->length);
+        tlv = (struct ndpi_packet_tlv *)&trailer->metadata[tot_len];
+      }
+      if(flow->ssh_tls.ja3_client[0] != '\0') {
+        tlv->type = ntohs(WIRESHARK_METADATA_JA3C);
+        tlv->length = ntohs(sizeof(flow->ssh_tls.ja3_client));
+        memcpy(tlv->data, flow->ssh_tls.ja3_client, sizeof(flow->ssh_tls.ja3_client));
+        /* TODO: boundary check */
+        tot_len += 4 + htons(tlv->length);
+        tlv = (struct ndpi_packet_tlv *)&trailer->metadata[tot_len];
+      }
+      if(flow->ssh_tls.ja3_server[0] != '\0') {
+        tlv->type = ntohs(WIRESHARK_METADATA_JA3S);
+        tlv->length = ntohs(sizeof(flow->ssh_tls.ja3_server));
+        memcpy(tlv->data, flow->ssh_tls.ja3_server, sizeof(flow->ssh_tls.ja3_server));
+        /* TODO: boundary check */
+        tot_len += 4 + htons(tlv->length);
+        tlv = (struct ndpi_packet_tlv *)&trailer->metadata[tot_len];
+      }
+      if(flow->ssh_tls.ja4_client[0] != '\0') {
+        tlv->type = ntohs(WIRESHARK_METADATA_JA4C);
+        tlv->length = ntohs(sizeof(flow->ssh_tls.ja4_client));
+        memcpy(tlv->data, flow->ssh_tls.ja4_client, sizeof(flow->ssh_tls.ja4_client));
+        /* TODO: boundary check */
+        tot_len += 4 + htons(tlv->length);
+        tlv = (struct ndpi_packet_tlv *)&trailer->metadata[tot_len];
+      }
+
+      flow->detection_completed = 2; /* Avoid exporting metadata again.
+                                        If we really want to have the metadata on Wireshark for *all*
+                                        the future packets of this flow, simply remove that assignment */
+    }
+    /* Last: padding */
+    tlv->type = 0;
+    tlv->length = ntohs(WIRESHARK_METADATA_SIZE - tot_len - 4);
+    /* The remaining bytes are already set to 0 */
+
     crc = (uint32_t*)&extcap_buf[h.caplen+sizeof(struct ndpi_packet_trailer)];
     *crc = ndpi_crc32((const void*)extcap_buf, h.caplen+sizeof(struct ndpi_packet_trailer));
     h.caplen += delta, h.len += delta;
diff --git a/example/reader_util.c b/example/reader_util.c
index 72e1843ae..cc74778e3 100644
--- a/example/reader_util.c
+++ b/example/reader_util.c
@@ -1440,7 +1440,8 @@ static struct ndpi_proto packet_processing(struct ndpi_workflow * workflow,
 					   const struct pcap_pkthdr *header,
 					   const u_char *packet,
 					   pkt_timeval when,
-					   ndpi_risk *flow_risk) {
+					   ndpi_risk *flow_risk,
+					   struct ndpi_flow_info **flow_ext) {
   struct ndpi_flow_info *flow = NULL;
   struct ndpi_flow_struct *ndpi_flow = NULL;
   u_int8_t proto;
@@ -1702,6 +1703,7 @@ static struct ndpi_proto packet_processing(struct ndpi_workflow * workflow,
 #endif
 
   *flow_risk = flow->risk;
+  *flow_ext = flow;
 
   return(flow->detected_protocol);
 }
@@ -1851,7 +1853,8 @@ static uint32_t ndpi_is_valid_gre_tunnel(const struct pcap_pkthdr *header,
 struct ndpi_proto ndpi_workflow_process_packet(struct ndpi_workflow * workflow,
 					       const struct pcap_pkthdr *header,
 					       const u_char *packet,
-					       ndpi_risk *flow_risk) {
+					       ndpi_risk *flow_risk,
+					       struct ndpi_flow_info **flow) {
   /*
    * Declare pointers to packet headers
    */
@@ -1900,6 +1903,7 @@ struct ndpi_proto ndpi_workflow_process_packet(struct ndpi_workflow * workflow,
   u_int8_t vlan_packet = 0;
 
   *flow_risk = 0 /* NDPI_NO_RISK */;
+  *flow = NULL;
 
   /* Increment raw packet counter */
   workflow->stats.raw_packet_count++;
@@ -2376,7 +2380,7 @@ struct ndpi_proto ndpi_workflow_process_packet(struct ndpi_workflow * workflow,
   return(packet_processing(workflow, time_ms, vlan_id, tunnel_type, iph, iph6,
 			   header->caplen - ip_offset,
 			   header->caplen, header, packet, header->ts,
-			   flow_risk));
+			   flow_risk, flow));
 }
 
 /* *********************************************** */
diff --git a/example/reader_util.h b/example/reader_util.h
index 79db35b03..945b88d84 100644
--- a/example/reader_util.h
+++ b/example/reader_util.h
@@ -416,7 +416,8 @@ void ndpi_free_flow_info_half(struct ndpi_flow_info *flow);
 struct ndpi_proto ndpi_workflow_process_packet(struct ndpi_workflow * workflow,
 					       const struct pcap_pkthdr *header,
 					       const u_char *packet,
-					       ndpi_risk *flow_risk);
+					       ndpi_risk *flow_risk,
+					       struct ndpi_flow_info **flow);
 
 
 /* Flow callback for completed flows, before the flow memory will be freed. */
diff --git a/fuzz/fuzz_ndpi_reader.c b/fuzz/fuzz_ndpi_reader.c
index 80ed16886..41e7be063 100644
--- a/fuzz/fuzz_ndpi_reader.c
+++ b/fuzz/fuzz_ndpi_reader.c
@@ -126,9 +126,10 @@ int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) {
 
     if(packet_checked) {
       ndpi_risk flow_risk;
+      struct ndpi_flow_info *flow = NULL; /* unused */
 
       memcpy(packet_checked, pkt, header->caplen);
-      ndpi_workflow_process_packet(workflow, header, packet_checked, &flow_risk);
+      ndpi_workflow_process_packet(workflow, header, packet_checked, &flow_risk, &flow);
       free(packet_checked);
     }
 
diff --git a/fuzz/fuzz_readerutils_workflow.cpp b/fuzz/fuzz_readerutils_workflow.cpp
index 9087a72c1..3517003b6 100644
--- a/fuzz/fuzz_readerutils_workflow.cpp
+++ b/fuzz/fuzz_readerutils_workflow.cpp
@@ -23,6 +23,7 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
   ndpi_serialization_format serialization_format;
   NDPI_PROTOCOL_BITMASK enabled_bitmask;
   ndpi_risk flow_risk;
+  struct ndpi_flow_info *flow = NULL; /* unused */
   const u_char *pkt;
   struct pcap_pkthdr *header;
   int r, rc;
@@ -95,7 +96,7 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
       header = NULL;
       r = pcap_next_ex(pcap_handle, &header, &pkt);
       while (r > 0) {
-        ndpi_workflow_process_packet(w, header, pkt, &flow_risk);
+        ndpi_workflow_process_packet(w, header, pkt, &flow_risk, &flow);
         r = pcap_next_ex(pcap_handle, &header, &pkt);
       }
     }
diff --git a/wireshark/ndpi.lua b/wireshark/ndpi.lua
index 3b10418c1..fb3af5945 100644
--- a/wireshark/ndpi.lua
+++ b/wireshark/ndpi.lua
@@ -38,6 +38,25 @@ ndpi_fds.name                 = ProtoField.new("nDPI Protocol Name", "ndpi.proto
 ndpi_fds.flow_risk            = ProtoField.new("nDPI Flow Risk", "ndpi.flow_risk", ftypes.UINT64, nil, base.HEX)
 ndpi_fds.flow_score           = ProtoField.new("nDPI Flow Score", "ndpi.flow_score", ftypes.UINT32)
 
+ndpi_fds.metadata_list        = ProtoField.new("nDPI Metadata List", "ndpi.metadata_list", ftypes.NONE)
+ndpi_fds.metadata             = ProtoField.new("nDPI Metadata", "ndpi.metadata", ftypes.NONE)
+local mtd_types = {
+        [0] = "Padding",
+        [1] = "Server Name",
+        [2] = "JA3C",
+        [3] = "JA3S",
+        [4] = "JA4C"
+}
+ndpi_fds.metadata_type        = ProtoField.new("nDPI Metadata Type", "ndpi.metadata.type", ftypes.UINT16, mtd_types)
+ndpi_fds.metadata_length      = ProtoField.new("nDPI Metadata Length", "ndpi.metadata.length", ftypes.UINT16)
+-- Generic field
+ndpi_fds.metadata_value       = ProtoField.new("nDPI Metadata Value", "ndpi.metadata.value", ftypes.BYTES)
+-- Specific fields
+ndpi_fds.metadata_server_name = ProtoField.new("nDPI Server Name", "ndpi.metadata.server_name", ftypes.STRING)
+ndpi_fds.metadata_ja3c        = ProtoField.new("nDPI JA3C", "ndpi.metadata.ja3c", ftypes.STRING)
+ndpi_fds.metadata_ja3s        = ProtoField.new("nDPI JA3S", "ndpi.metadata.ja3s", ftypes.STRING)
+ndpi_fds.metadata_ja4c        = ProtoField.new("nDPI JA4C", "ndpi.metadata.ja4c", ftypes.STRING)
+
 
 local flow_risks = {}
 --- You can't use a 64 bit integer "as-is" as mask: we choose to use UInt64 object instead
@@ -1055,9 +1074,9 @@ function ndpi_proto.dissector(tvb, pinfo, tree)
 
 	 if(magic == "19:68:09:24") then
 	    local ndpikey, srckey, dstkey, flowkey, flow_risk
-	    local flow_risk_tree
+	    local flow_risk_tree, metadata_list_tree, metadata_tree
 	    local name
-	    local trailer_tvb          = tvb(tvb:len() - 38, 34) -- The last 4 bytes are the CRC. Even if nDPI needs to update it, it is not part of the nDPI trailer, strictly speaking
+	    local trailer_tvb          = tvb(tvb:len() - 294 , 290) -- The last 4 bytes are the CRC. Even if nDPI needs to update it, it is not part of the nDPI trailer, strictly speaking
 	    local ndpi_subtree         = tree:add(ndpi_proto, trailer_tvb, "nDPI Protocol")
 	    
 	    ndpi_subtree:add(ndpi_fds.magic, trailer_tvb(0, 4))
@@ -1113,6 +1132,39 @@ function ndpi_proto.dissector(tvb, pinfo, tree)
 	       --print(network_protocol .. "/" .. application_protocol .. "/".. name)
 	    end
 
+	    -- Metadata
+	    local offset = 34
+	    metadata_list_tree = ndpi_subtree:add(ndpi_fds.metadata_list, trailer_tvb(offset, 256))
+	    while offset + 4 < 294 do
+
+	       local mtd_type = trailer_tvb(offset, 2):int();
+	       local mtd_length = trailer_tvb(offset + 2, 2):int();
+
+	       metadata_tree = metadata_list_tree:add(ndpi_fds.metadata, trailer_tvb(offset, 4 + mtd_length))
+	       metadata_tree:add(ndpi_fds.metadata_type, trailer_tvb(offset, 2))
+	       metadata_tree:add(ndpi_fds.metadata_length, trailer_tvb(offset + 2, 2))
+
+	       -- Specific fields: there is definitely a better way...
+	       if mtd_type == 1 then
+	         metadata_tree:append_text(" ServerName: " .. trailer_tvb(offset + 4, mtd_length):string())
+	         metadata_tree:add(ndpi_fds.metadata_server_name, trailer_tvb(offset + 4, mtd_length))
+	       elseif mtd_type == 2 then
+	         metadata_tree:append_text(" JA3C: " .. trailer_tvb(offset + 4, mtd_length):string())
+	         metadata_tree:add(ndpi_fds.metadata_ja3c, trailer_tvb(offset + 4, mtd_length))
+	       elseif mtd_type == 3 then
+	         metadata_tree:append_text(" JA3S: " .. trailer_tvb(offset + 4, mtd_length):string())
+	         metadata_tree:add(ndpi_fds.metadata_ja3s, trailer_tvb(offset + 4, mtd_length))
+	       elseif mtd_type == 4 then
+	         metadata_tree:append_text(" JA4C: " .. trailer_tvb(offset + 4, mtd_length):string())
+	         metadata_tree:add(ndpi_fds.metadata_ja4c, trailer_tvb(offset + 4, mtd_length))
+	       else
+	         -- Generic field
+	         metadata_tree:add(ndpi_fds.metadata_value, trailer_tvb(offset + 4, mtd_length))
+	       end
+
+	       offset = offset + 4 + mtd_length
+	    end
+
 	    if(compute_flows_stats and pinfo.visited == false) then
 	       ndpikey = tostring(slen(name))