Adds some risk exceptions for popular services and domain names

via a new (internal) function named ndpi_add_domain_risk_exceptions()

3 files changed, 42 insertions, 5 deletions

diff --git a/example/protos.txt b/example/protos.txt
index 3428ee109..7f47009bd 100644
--- a/example/protos.txt
+++ b/example/protos.txt
@@ -51,8 +51,7 @@ ip:54.80.47.130@AmazonPrime
 # For Flows with a hostname (e.g. TLS) the risk is also put in AND with the host_risk_mask
 #ip_risk_mask:192.168.1.0/24=0
 #ip_risk_mask:10.196.157.228=0
-host_risk_mask:".local"=0
-host_risk_mask:".msftconnecttest.com"=0
+host_risk_mask:".home"=0
 
 
 # Custom certification autorities we trust
diff --git a/src/lib/ndpi_main.c b/src/lib/ndpi_main.c
index 20df63de6..16f7b9ea0 100644
--- a/src/lib/ndpi_main.c
+++ b/src/lib/ndpi_main.c
@@ -761,6 +761,8 @@ static void ndpi_xgrams_init(unsigned int *dst,size_t dn, const char **src,size_
   }
 }
 
+/* ******************************************************************** */
+
 static void init_string_based_protocols(struct ndpi_detection_module_struct *ndpi_str) {
   int i;
 
@@ -2481,9 +2483,45 @@ struct ndpi_detection_module_struct *ndpi_init_detection_module(ndpi_init_prefs
 
 /* *********************************************** */
 
+/*
+  This function adds some exceptions for popular domain names
+  in order to avoid "false" positives and avoid polluting
+  results
+*/
+static void ndpi_add_domain_risk_exceptions(struct ndpi_detection_module_struct *ndpi_str) {
+  const char *domains[] = {
+    ".local",
+    ".msftconnecttest.com",
+    "amupdatedl.microsoft.com",
+    "update.microsoft.com.akadns.net",
+    ".windowsupdate.com",
+    "e5.sk",
+    "sophosxl.net",
+    NULL /* End */
+  };
+  const ndpi_risk risks_to_mask[] = {
+    NDPI_SUSPICIOUS_DGA_DOMAIN,
+    NDPI_BINARY_APPLICATION_TRANSFER,
+    NDPI_HTTP_NUMERIC_IP_HOST,
+    NDPI_NO_RISK /* End */
+  };
+  u_int i;
+  ndpi_risk mask = ((ndpi_risk)-1);
+
+  for(i=0; risks_to_mask[i] != NDPI_NO_RISK; i++)
+    mask &= ~(1UL << risks_to_mask[i]);
+
+  for(i=0; domains[i] != NULL; i++)
+    ndpi_add_host_risk_mask(ndpi_str, (char*)domains[i], mask);
+}
+
+/* *********************************************** */
+
 void ndpi_finalize_initialization(struct ndpi_detection_module_struct *ndpi_str) {
   u_int i;
 
+  ndpi_add_domain_risk_exceptions(ndpi_str);
+  
   if(ndpi_str->ac_automa_finalized) return;
 
   for(i = 0; i < 99; i++) {
diff --git a/tests/result/iphone.pcap.out b/tests/result/iphone.pcap.out
index c40b49273..ffa15c0c0 100644
--- a/tests/result/iphone.pcap.out
+++ b/tests/result/iphone.pcap.out
@@ -39,9 +39,9 @@ JA3 Host Stats:
 	13	TCP 192.168.2.17:50577 <-> 17.130.2.46:443 [proto: 91.140/TLS.Apple][Encrypted][Confidence: DPI][cat: Web/5][10 pkts/1721 bytes <-> 8 pkts/4801 bytes][Goodput ratio: 61/89][0.67 sec][Hostname/SNI: gsp85-ssl.ls.apple.com][ALPN: h2;http/1.1][TLS Supported Versions: TLSv1.3;TLSv1.2;TLSv1.1;TLSv1][bytes ratio: -0.472 (Download)][IAT c2s/s2c min/avg/max/stddev: 0/0 81/52 171/161 80/73][Pkt Len c2s/s2c min/avg/max/stddev: 66/66 172/600 583/1506 165/572][TLSv1.2][JA3C: 55271a105172d5f225e4704755b9b250][ServerNames: *.ls.apple.com][JA3S: 4ef1b297bb817d8212165a86308bac5f][Issuer: CN=Apple IST CA 2 - G1, OU=Certification Authority, O=Apple Inc., C=US][Subject: CN=*.ls.apple.com, OU=management:idms.group.576486, O=Apple Inc., ST=California, C=US][Certificate SHA-1: E4:85:25:4C:99:F8:FB:66:49:4B:80:64:5E:63:2A:75:9B:8F:C3:51][Safari][Validity: 2019-03-15 23:17:29 - 2021-04-13 23:17:29][Cipher: TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256][Plen Bins: 0,0,11,0,11,0,0,0,11,11,0,0,11,0,0,0,11,0,0,0,0,0,11,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,22,0,0]
 	14	TCP 192.168.2.17:50585 <-> 17.137.166.35:443 [proto: 91.140/TLS.Apple][Encrypted][Confidence: DPI][cat: Web/5][6 pkts/1051 bytes <-> 6 pkts/4246 bytes][Goodput ratio: 61/90][1.05 sec][Hostname/SNI: gsa.apple.com][ALPN: http/1.1][TLS Supported Versions: TLSv1.3;TLSv1.2;TLSv1.1;TLSv1][bytes ratio: -0.603 (Download)][IAT c2s/s2c min/avg/max/stddev: 0/0 132/52 322/206 138/89][Pkt Len c2s/s2c min/avg/max/stddev: 66/66 175/708 583/1506 188/647][TLSv1.2][JA3C: 6fa3244afc6bb6f9fad207b6b52af26b][ServerNames: gsas.apple.com,gsa.apple.com][JA3S: c4b2785a87896e19d37eee932070cb22][Issuer: CN=Apple Server Authentication CA, OU=Certification Authority, O=Apple Inc., C=US][Subject: CN=gsa.apple.com, O=Apple Inc., ST=California, C=US][Certificate SHA-1: D4:EF:5E:AD:7F:D5:13:5B:9F:B2:B9:84:19:75:BB:ED:53:FB:18:D6][Safari][Validity: 2019-03-07 00:55:40 - 2020-04-05 00:55:40][Cipher: TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256][Plen Bins: 0,16,0,16,0,0,0,0,0,0,0,0,0,0,0,0,16,0,0,0,0,0,0,0,0,0,0,0,16,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,34,0,0]
 	15	UDP 0.0.0.0:68 -> 255.255.255.255:67 [proto: 18/DHCP][ClearText][Confidence: DPI][cat: Network/14][7 pkts/2394 bytes -> 0 pkts/0 bytes][Goodput ratio: 88/0][43.15 sec][Hostname/SNI: lucas-imac][bytes ratio: 1.000 (Upload)][IAT c2s/s2c min/avg/max/stddev: 1022/0 7191/0 8962/0 2834/0][Pkt Len c2s/s2c min/avg/max/stddev: 342/0 342/0 342/0 0/0][DHCP Fingerprint: 1,121,3,6,15,119,252,95,44,46][PLAIN TEXT (iPhone)][Plen Bins: 0,0,0,0,0,0,0,0,0,100,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
-	16	UDP 169.254.225.216:5353 -> 224.0.0.251:5353 [proto: 8/MDNS][ClearText][Confidence: DPI][cat: Network/14][4 pkts/2123 bytes -> 0 pkts/0 bytes][Goodput ratio: 92/0][33.08 sec][Hostname/SNI: luca???s_imac._odisk._tcp.local][luca???s_imac._odisk._tcp.local][PLAIN TEXT (s iMac)][Plen Bins: 0,25,0,0,0,0,0,0,0,0,0,50,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,25,0,0,0,0,0,0,0,0,0,0]
-	17	UDP 192.168.2.1:5353 -> 224.0.0.251:5353 [proto: 8/MDNS][ClearText][Confidence: DPI][cat: Network/14][4 pkts/2094 bytes -> 0 pkts/0 bytes][Goodput ratio: 92/0][33.08 sec][Hostname/SNI: luca???s_imac._odisk._tcp.local][luca???s_imac._odisk._tcp.local][PLAIN TEXT (s iMac)][Plen Bins: 0,25,0,0,0,0,0,0,0,0,0,50,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,25,0,0,0,0,0,0,0,0,0,0,0]
-	18	UDP [fe80::c42c:3ff:fe60:6a64]:5353 -> [ff02::fb]:5353 [proto: 8/MDNS][ClearText][Confidence: DPI][cat: Network/14][3 pkts/2067 bytes -> 0 pkts/0 bytes][Goodput ratio: 91/0][33.08 sec][Hostname/SNI: luca???s_imac._odisk._tcp.local][luca???s_imac._odisk._tcp.local][PLAIN TEXT (s iMac)][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,66,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,33,0,0,0,0,0,0,0,0,0,0,0]
+	16	UDP 169.254.225.216:5353 -> 224.0.0.251:5353 [proto: 8/MDNS][ClearText][Confidence: DPI][cat: Network/14][4 pkts/2123 bytes -> 0 pkts/0 bytes][Goodput ratio: 92/0][33.08 sec][Hostname/SNI: luca???s_imac._odisk._tcp.local][luca???s_imac._odisk._tcp.local][Risk: ** Text contains non-printable characters **][Risk Score: 100][PLAIN TEXT (s iMac)][Plen Bins: 0,25,0,0,0,0,0,0,0,0,0,50,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,25,0,0,0,0,0,0,0,0,0,0]
+	17	UDP 192.168.2.1:5353 -> 224.0.0.251:5353 [proto: 8/MDNS][ClearText][Confidence: DPI][cat: Network/14][4 pkts/2094 bytes -> 0 pkts/0 bytes][Goodput ratio: 92/0][33.08 sec][Hostname/SNI: luca???s_imac._odisk._tcp.local][luca???s_imac._odisk._tcp.local][Risk: ** Text contains non-printable characters **][Risk Score: 100][PLAIN TEXT (s iMac)][Plen Bins: 0,25,0,0,0,0,0,0,0,0,0,50,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,25,0,0,0,0,0,0,0,0,0,0,0]
+	18	UDP [fe80::c42c:3ff:fe60:6a64]:5353 -> [ff02::fb]:5353 [proto: 8/MDNS][ClearText][Confidence: DPI][cat: Network/14][3 pkts/2067 bytes -> 0 pkts/0 bytes][Goodput ratio: 91/0][33.08 sec][Hostname/SNI: luca???s_imac._odisk._tcp.local][luca???s_imac._odisk._tcp.local][Risk: ** Text contains non-printable characters **][Risk Score: 100][PLAIN TEXT (s iMac)][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,66,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,33,0,0,0,0,0,0,0,0,0,0,0]
 	19	TCP 192.168.2.17:49152 <-> 17.253.105.202:80 [proto: 7.140/HTTP.Apple][ClearText][Confidence: DPI][cat: ConnCheck/30][5 pkts/473 bytes <-> 4 pkts/968 bytes][Goodput ratio: 28/72][0.33 sec][Hostname/SNI: captive.apple.com][bytes ratio: -0.344 (Download)][IAT c2s/s2c min/avg/max/stddev: 0/2 82/80 171/158 82/78][Pkt Len c2s/s2c min/avg/max/stddev: 66/66 95/242 197/762 51/300][URL: captive.apple.com/hotspot-detect.html][StatusCode: 200][Content-Type: text/html][User-Agent: CaptiveNetworkSupport-390.60.1 wispr][PLAIN TEXT (GET /hotspot)][Plen Bins: 0,0,0,0,50,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,50,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
 	20	UDP 192.168.2.1:17500 -> 192.168.2.255:17500 [proto: 121/Dropbox][ClearText][Confidence: DPI][cat: Cloud/13][2 pkts/1104 bytes -> 0 pkts/0 bytes][Goodput ratio: 92/0][30.05 sec][PLAIN TEXT (version)][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,100,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
 	21	UDP 192.168.2.1:67 -> 192.168.2.17:68 [proto: 18/DHCP][ClearText][Confidence: DPI][cat: Network/14][2 pkts/684 bytes -> 0 pkts/0 bytes][Goodput ratio: 88/0][1.02 sec][PLAIN TEXT (iMac.local)][Plen Bins: 0,0,0,0,0,0,0,0,0,100,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]