ndpireader: fix detection of DoH traffic based on packet distributions (#2045)

6 files changed, 105 insertions, 24 deletions

diff --git a/example/ndpiReader.c b/example/ndpiReader.c
index 053cfe38b..dfde22d61 100644
--- a/example/ndpiReader.c
+++ b/example/ndpiReader.c
@@ -269,33 +269,37 @@ FILE *trace = NULL;
 
 #define NUM_DOH_BINS 2
 
-struct ndpi_bin doh_ndpi_bins[NUM_DOH_BINS];
+static struct ndpi_bin doh_ndpi_bins[NUM_DOH_BINS];
 
-u_int8_t doh_centroids[NUM_DOH_BINS][PLEN_NUM_BINS] = {
+static u_int8_t doh_centroids[NUM_DOH_BINS][PLEN_NUM_BINS] = {
   { 23,25,3,0,26,0,0,0,0,0,0,0,0,0,2,0,0,15,3,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 },
   { 35,30,21,0,0,0,2,4,0,0,5,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 }
 };
 
-float doh_max_distance = 35.5;
+static float doh_max_distance = 35.5;
 
-void init_doh_bins() {
+static void init_doh_bins() {
   u_int i;
 
   for(i=0; i<NUM_DOH_BINS; i++) {
     ndpi_init_bin(&doh_ndpi_bins[i], ndpi_bin_family8, PLEN_NUM_BINS);
+    ndpi_free_bin(&doh_ndpi_bins[i]); /* Hack: we use static bins (see below), so we need to free the dynamic ones just allocated */
     doh_ndpi_bins[i].u.bins8 = doh_centroids[i];
   }
 }
 
 /* *********************************************** */
 
-u_int check_bin_doh_similarity(struct ndpi_bin *bin, float *similarity) {
+static u_int check_bin_doh_similarity(struct ndpi_bin *bin, float *similarity) {
   u_int i;
   float lowest_similarity = 9999999999.0f;
 
   for(i=0; i<NUM_DOH_BINS; i++) {
     *similarity = ndpi_bin_similarity(&doh_ndpi_bins[i], bin, 0, 0);
 
+    if(*similarity < 0) /* Error */
+      return(0);
+
     if(*similarity <= doh_max_distance)
       return(1);
 
@@ -3402,7 +3406,7 @@ static void printFlowsStats() {
 
           ndpi_cluster_bins(bins, num_flow_bins, num_bin_clusters, cluster_ids, centroids);
 
-          printf("\n"
+          fprintf(out, "\n"
                  "\tBin clusters\n"
                  "\t------------\n");
 
@@ -3416,23 +3420,23 @@ static void printFlowsStats() {
               if(cluster_ids[i] != j) continue;
 
               if(num_printed == 0) {
-                printf("\tCluster %u [", j);
+                fprintf(out, "\tCluster %u [", j);
                 print_bin(out, NULL, &centroids[j]);
-                printf("]\n");
+                fprintf(out, "]\n");
               }
 
-              printf("\t%u\t%-10s\t%s:%u <-> %s:%u\t[",
-                     i,
-                     ndpi_protocol2name(ndpi_thread_info[0].workflow->ndpi_struct,
-                                        all_flows[i].flow->detected_protocol, buf, sizeof(buf)),
-                     all_flows[i].flow->src_name,
-                     ntohs(all_flows[i].flow->src_port),
-                     all_flows[i].flow->dst_name,
-                     ntohs(all_flows[i].flow->dst_port));
+              fprintf(out, "\t%u\t%-10s\t%s:%u <-> %s:%u\t[",
+                      i,
+                      ndpi_protocol2name(ndpi_thread_info[0].workflow->ndpi_struct,
+                                         all_flows[i].flow->detected_protocol, buf, sizeof(buf)),
+                      all_flows[i].flow->src_name,
+                      ntohs(all_flows[i].flow->src_port),
+                      all_flows[i].flow->dst_name,
+                      ntohs(all_flows[i].flow->dst_port));
 
               print_bin(out, NULL, &bins[i]);
-              printf("][similarity: %f]",
-                     (similarity = ndpi_bin_similarity(&centroids[j], &bins[i], 0, 0)));
+              fprintf(out, "][similarity: %f]",
+                      (similarity = ndpi_bin_similarity(&centroids[j], &bins[i], 0, 0)));
 
               if(all_flows[i].flow->host_server_name[0] != '\0')
                 fprintf(out, "[%s]", all_flows[i].flow->host_server_name);
@@ -3445,23 +3449,23 @@ static void printFlowsStats() {
                    && all_flows[i].flow->ssh_tls.advertised_alpns /* ALPN */
                    ) {
                   if(check_bin_doh_similarity(&bins[i], &s))
-                    printf("[DoH (%f distance)]", s);
+                    fprintf(out, "[DoH (%f distance)]", s);
                   else
-                    printf("[NO DoH (%f distance)]", s);
+                    fprintf(out, "[NO DoH (%f distance)]", s);
                 } else {
                   if(all_flows[i].flow->ssh_tls.advertised_alpns == NULL)
-                    printf("[NO DoH check: missing ALPN]");
+                    fprintf(out, "[NO DoH check: missing ALPN]");
                 }
               }
 
-              printf("\n");
+              fprintf(out, "\n");
               num_printed++;
               if(similarity > max_similarity) max_similarity = similarity;
             }
 
             if(num_printed) {
-              printf("\tMax similarity: %f\n", max_similarity);
-              printf("\n");
+              fprintf(out, "\tMax similarity: %f\n", max_similarity);
+              fprintf(out, "\n");
             }
           }
 
@@ -5414,6 +5418,14 @@ int main(int argc, char **argv) {
     exit(0);
   }
 
+  if(enable_doh_dot_detection) {
+    init_doh_bins();
+    /* Clusters are not really used in DoH/DoT detection, but because of how
+       the code has been written, we need to enable also clustering feature */
+    if(num_bin_clusters == 0)
+      num_bin_clusters = 1;
+  }
+
   if(!quiet_mode) {
     printf("\n-----------------------------------------------------------\n"
 	   "* NOTE: This is demo app to show *some* nDPI features.\n"
diff --git a/tests/cfgs/default/pcap/doh.pcapng b/tests/cfgs/default/pcap/doh.pcapng
new file mode 100644
index 000000000..cdc166f9c
--- /dev/null
+++ b/tests/cfgs/default/pcap/doh.pcapng
diff --git a/tests/cfgs/default/result/doh.pcapng.out b/tests/cfgs/default/result/doh.pcapng.out
new file mode 100644
index 000000000..31df8bc95
--- /dev/null
+++ b/tests/cfgs/default/result/doh.pcapng.out
@@ -0,0 +1,30 @@
+Guessed flow protos:	0
+
+DPI Packets (TCP):	6	(6.00 pkts/flow)
+Confidence DPI              : 1 (flows)
+Num dissector calls: 1 (1.00 diss/flow)
+LRU cache ookla:      0/0/0 (insert/search/found)
+LRU cache bittorrent: 0/0/0 (insert/search/found)
+LRU cache zoom:       0/0/0 (insert/search/found)
+LRU cache stun:       0/0/0 (insert/search/found)
+LRU cache tls_cert:   0/2/0 (insert/search/found)
+LRU cache mining:     0/0/0 (insert/search/found)
+LRU cache msteams:    0/0/0 (insert/search/found)
+LRU cache stun_zoom:  0/0/0 (insert/search/found)
+Automa host:          0/0 (search/found)
+Automa domain:        0/0 (search/found)
+Automa tls cert:      0/0 (search/found)
+Automa risk mask:     0/0 (search/found)
+Automa common alpns:  2/2 (search/found)
+Patricia risk mask:   2/0 (search/found)
+Patricia risk:        0/0 (search/found)
+Patricia protocols:   2/0 (search/found)
+
+TLS	120	14592	1
+
+JA3 Host Stats: 
+		 IP Address                  	 # JA3C     
+	1	 192.168.1.253            	 1      
+
+
+	1	TCP 192.168.1.253:35996 <-> 1.1.1.1:443 [proto: 91/TLS][IP: 0/Unknown][Encrypted][Confidence: DPI][DPI packets: 6][cat: Web/5][61 pkts/5381 bytes <-> 59 pkts/9211 bytes][Goodput ratio: 35/63][122.79 sec][(Advertised) ALPNs: h2;http/1.1][TLS Supported Versions: TLSv1.3;TLSv1.2;TLSv1.1;TLSv1][bytes ratio: -0.262 (Download)][IAT c2s/s2c min/avg/max/stddev: 0/0 1965/1934 15360/15360 4993/4853][Pkt Len c2s/s2c min/avg/max/stddev: 60/60 88/156 315/1514 41/267][Risk: ** Missing SNI TLS Extn **][Risk Score: 50][TLSv1.3][JA3C: 7c1e207beb00684bbbe144f1b0abe1d5][JA3S: d75f9129bb5d05492a65ff78e081bcb2][Firefox][Cipher: TLS_CHACHA20_POLY1305_SHA256][Plen Bins: 22,26,24,1,1,7,5,5,1,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,3,0,0]
diff --git a/tests/cfgs/enable_doh_heuristic/config.txt b/tests/cfgs/enable_doh_heuristic/config.txt
new file mode 100644
index 000000000..eb11be000
--- /dev/null
+++ b/tests/cfgs/enable_doh_heuristic/config.txt
@@ -0,0 +1 @@
+-D
diff --git a/tests/cfgs/enable_doh_heuristic/pcap/doh.pcapng b/tests/cfgs/enable_doh_heuristic/pcap/doh.pcapng
new file mode 120000
index 000000000..d03d021aa
--- /dev/null
+++ b/tests/cfgs/enable_doh_heuristic/pcap/doh.pcapng
@@ -0,0 +1 @@
+../../default/pcap/doh.pcapng
\ No newline at end of file
diff --git a/tests/cfgs/enable_doh_heuristic/result/doh.pcapng.out b/tests/cfgs/enable_doh_heuristic/result/doh.pcapng.out
new file mode 100644
index 000000000..d301dba24
--- /dev/null
+++ b/tests/cfgs/enable_doh_heuristic/result/doh.pcapng.out
@@ -0,0 +1,37 @@
+Guessed flow protos:	0
+
+DPI Packets (TCP):	24	(24.00 pkts/flow)
+Confidence DPI              : 1 (flows)
+Num dissector calls: 1 (1.00 diss/flow)
+LRU cache ookla:      0/0/0 (insert/search/found)
+LRU cache bittorrent: 0/0/0 (insert/search/found)
+LRU cache zoom:       0/0/0 (insert/search/found)
+LRU cache stun:       0/0/0 (insert/search/found)
+LRU cache tls_cert:   0/2/0 (insert/search/found)
+LRU cache mining:     0/0/0 (insert/search/found)
+LRU cache msteams:    0/0/0 (insert/search/found)
+LRU cache stun_zoom:  0/0/0 (insert/search/found)
+Automa host:          0/0 (search/found)
+Automa domain:        0/0 (search/found)
+Automa tls cert:      0/0 (search/found)
+Automa risk mask:     0/0 (search/found)
+Automa common alpns:  2/2 (search/found)
+Patricia risk mask:   2/0 (search/found)
+Patricia risk:        0/0 (search/found)
+Patricia protocols:   2/0 (search/found)
+
+TLS	120	14592	1
+
+JA3 Host Stats: 
+		 IP Address                  	 # JA3C     
+	1	 192.168.1.253            	 1      
+
+
+	1	TCP 192.168.1.253:35996 <-> 1.1.1.1:443 [proto: 91/TLS][IP: 0/Unknown][Encrypted][Confidence: DPI][DPI packets: 24][cat: Web/5][61 pkts/5381 bytes <-> 59 pkts/9211 bytes][Goodput ratio: 35/63][122.79 sec][(Advertised) ALPNs: h2;http/1.1][TLS Supported Versions: TLSv1.3;TLSv1.2;TLSv1.1;TLSv1][bytes ratio: -0.262 (Download)][IAT c2s/s2c min/avg/max/stddev: 0/0 1965/1934 15360/15360 4993/4853][Pkt Len c2s/s2c min/avg/max/stddev: 60/60 88/156 315/1514 41/267][Risk: ** Missing SNI TLS Extn **][Risk Score: 50][TLSv1.3][JA3C: 7c1e207beb00684bbbe144f1b0abe1d5][JA3S: d75f9129bb5d05492a65ff78e081bcb2][Firefox][Cipher: TLS_CHACHA20_POLY1305_SHA256][Plen Bins: 24,32,24,0,1,7,3,5,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
+
+	Bin clusters
+	------------
+	Cluster 0 [24;32;24;0;1;7;3;5;0;0;1;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0]
+	0	TLS       	192.168.1.253:35996 <-> 1.1.1.1:443	[24;32;24;0;1;7;3;5;0;0;1;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0][similarity: 0.000000][DoH (14.247807 distance)]
+	Max similarity: 0.000000
+