Added checks for DGA detection

9 files changed, 34 insertions, 27 deletions

diff --git a/example/intrusion_detection.c b/example/intrusion_detection.c
index 7296c2d29..ef4189a56 100644
--- a/example/intrusion_detection.c
+++ b/example/intrusion_detection.c
@@ -1,7 +1,7 @@
 /*
  * intrusion_detection.c
  *
- * Copyright (C) 2011-19 - ntop.org
+ * Copyright (C) 2011-20 - ntop.org
  *
  * This file is part of nDPI, an open source deep packet inspection
  * library based on the OpenDPI and PACE technology by ipoque GmbH
diff --git a/example/ndpiReader.c b/example/ndpiReader.c
index 8ce4c6263..f25794795 100644
--- a/example/ndpiReader.c
+++ b/example/ndpiReader.c
@@ -1,7 +1,7 @@
 /*
  * ndpiReader.c
  *
- * Copyright (C) 2011-19 - ntop.org
+ * Copyright (C) 2011-20 - ntop.org
  *
  * nDPI is free software: you can redistribute it and/or modify
  * it under the terms of the GNU Lesser General Public License as published by
diff --git a/example/reader_util.c b/example/reader_util.c
index b4e2c943a..be2f49b4f 100644
--- a/example/reader_util.c
+++ b/example/reader_util.c
@@ -1,7 +1,7 @@
 /*
  * reader_util.c
  *
- * Copyright (C) 2011-19 - ntop.org
+ * Copyright (C) 2011-20 - ntop.org
  *
  * This file is part of nDPI, an open source deep packet inspection
  * library based on the OpenDPI and PACE technology by ipoque GmbH
diff --git a/src/lib/ndpi_main.c b/src/lib/ndpi_main.c
index 325f33233..00ecadb41 100644
--- a/src/lib/ndpi_main.c
+++ b/src/lib/ndpi_main.c
@@ -6083,8 +6083,12 @@ int ndpi_match_bigram(struct ndpi_detection_module_struct *ndpi_str,
     return(-1);
 
   if(!automa->ac_automa_finalized) {
+#if 1
+    ndpi_finalize_initalization(ndpi_str);
+#else
     printf("[%s:%d] [NDPI] Internal error: please call ndpi_finalize_initalization()\n", __FILE__, __LINE__);
     return(0); /* No matches */
+#endif
   }
 
   ac_input_text.astring = bigram_to_match, ac_input_text.length = 2;
@@ -6483,12 +6487,12 @@ void ndpi_md5(const u_char *data, size_t data_len, u_char hash[16]) {
 
 static int enough(int a, int b) {
   u_int8_t percentage = 20;
-  
+
   if(b == 0) return(0);
   if(a == 0) return(1);
 
-  if(b > ((a*percentage)/100)) return(1);
-  
+  if(b > (((a+1)*percentage)/100)) return(1);
+
   return(0);
 }
 
@@ -6498,14 +6502,14 @@ int ndpi_check_dga_name(struct ndpi_detection_module_struct *ndpi_str,
 			struct ndpi_flow_struct *flow,
 			char *name) {
   int len = strlen(name), rc = 0;
- 
+
   if(len >= 5) {
     int i, j, num_found = 0, num_impossible = 0, num_bigram_checks = 0;
     char tmp[128];
 
     len = snprintf(tmp, sizeof(tmp)-1, "%s", name);
     if(len < 0) return(0);
-    
+
     for(i=0, j=0; (i<len) && (j<(sizeof(tmp)-1)); i++) {
       if(isdigit(name[i]))
 	continue;
@@ -6513,11 +6517,12 @@ int ndpi_check_dga_name(struct ndpi_detection_module_struct *ndpi_str,
 	tmp[j++] = tolower(name[i]);
     }
 
+    tmp[j] = '\0';
     len = j;
-    
-    for(i = 0; tmp[i+1] != '\0'; i++) {
+
+    for(i = 0; i < len; i++) {
       if(isdigit(tmp[i])) continue;
-	
+
       switch(tmp[i]) {
       case '-':
       case ':':
@@ -6527,7 +6532,7 @@ int ndpi_check_dga_name(struct ndpi_detection_module_struct *ndpi_str,
       }
 
       if(isdigit(tmp[i+1])) continue;
-	 
+
       num_bigram_checks++;
 
       if(ndpi_match_bigram(ndpi_str, &ndpi_str->bigrams_automa, &tmp[i])) {
@@ -6543,9 +6548,9 @@ int ndpi_check_dga_name(struct ndpi_detection_module_struct *ndpi_str,
     }
 
     if(num_bigram_checks
-       && ((num_found == 0)
-	   || (enough(num_found, num_impossible))))
-      rc = 1;    
+       && (num_impossible > 0)
+       && ((num_found == 0) || enough(num_found, num_impossible)))
+      rc = 1;
 
     if(rc && flow)
       NDPI_SET_BIT(flow->risk, NDPI_SUSPICIOUS_DGA_DOMAIN);
@@ -6556,6 +6561,6 @@ int ndpi_check_dga_name(struct ndpi_detection_module_struct *ndpi_str,
 	     tmp, name, num_found, num_impossible);
 #endif
   }
-  
+
   return(rc);
 }
diff --git a/src/lib/protocols/dns.c b/src/lib/protocols/dns.c
index 099f343e4..897fb1e4b 100644
--- a/src/lib/protocols/dns.c
+++ b/src/lib/protocols/dns.c
@@ -294,11 +294,12 @@ static void ndpi_search_dns(struct ndpi_detection_module_struct *ndpi_struct, st
     }
 
     flow->host_server_name[j] = '\0';
-    ndpi_check_dga_name(ndpi_struct, flow, (char*)flow->host_server_name);
     
     if(j > 0) {
       ndpi_protocol_match_result ret_match;
 
+      ndpi_check_dga_name(ndpi_struct, flow, (char*)flow->host_server_name);
+      
       ret.app_protocol = ndpi_match_host_subprotocol(ndpi_struct, flow,
 						     (char *)flow->host_server_name,
 						     strlen((const char*)flow->host_server_name),
diff --git a/src/lib/protocols/http.c b/src/lib/protocols/http.c
index a896c6214..7d54d1470 100644
--- a/src/lib/protocols/http.c
+++ b/src/lib/protocols/http.c
@@ -450,7 +450,7 @@ static void check_content_type_and_change_protocol(struct ndpi_detection_module_
     flow->host_server_name[len] = '\0';
     flow->extra_packets_func = NULL; /* We're good now */
 
-    ndpi_check_dga_name(ndpi_struct, flow, (char*)flow->host_server_name);
+    if(len > 0) ndpi_check_dga_name(ndpi_struct, flow, (char*)flow->host_server_name);
     flow->server_id = flow->dst;
 
     if(packet->forwarded_line.ptr) {
diff --git a/src/lib/protocols/netbios.c b/src/lib/protocols/netbios.c
index c396a392b..66224557a 100644
--- a/src/lib/protocols/netbios.c
+++ b/src/lib/protocols/netbios.c
@@ -80,8 +80,9 @@ static void ndpi_int_netbios_add_connection(struct ndpi_detection_module_struct
   char name[64];
   u_int off = flow->packet.payload[12] == 0x20 ? 12 : 14;
 
-  if((off < flow->packet.payload_packet_len) &&
-     ndpi_netbios_name_interpret((char*)&flow->packet.payload[off], flow->packet.payload_packet_len - off, name, sizeof(name)) > 0) {
+  if((off < flow->packet.payload_packet_len)
+     && ndpi_netbios_name_interpret((char*)&flow->packet.payload[off],
+				 flow->packet.payload_packet_len - off, name, sizeof(name)) > 0) {
       snprintf((char*)flow->host_server_name, sizeof(flow->host_server_name)-1, "%s", name);
 
       ndpi_check_dga_name(ndpi_struct, flow, (char*)flow->host_server_name);
diff --git a/tests/result/1kxun.pcap.out b/tests/result/1kxun.pcap.out
index ab0282e92..8c4ae196e 100644
--- a/tests/result/1kxun.pcap.out
+++ b/tests/result/1kxun.pcap.out
@@ -82,9 +82,9 @@ JA3 Host Stats:
 	60	UDP [fe80::9bd:81dd:2fdc:5750]:61548 -> [ff02::1:3]:5355 [proto: 154/LLMNR][cat: Network/14][2 pkts/190 bytes -> 0 pkts/0 bytes][Goodput ratio: 35/0][0.41 sec][Host: caesar-thinkpad][PLAIN TEXT (caesar)]
 	61	UDP [fe80::9bd:81dd:2fdc:5750]:64568 -> [ff02::1:3]:5355 [proto: 154/LLMNR][cat: Network/14][2 pkts/190 bytes -> 0 pkts/0 bytes][Goodput ratio: 35/0][0.41 sec][Host: caesar-thinkpad][PLAIN TEXT (caesar)]
 	62	UDP 192.168.5.45:137 -> 192.168.255.255:137 [proto: 10/NetBIOS][cat: System/18][2 pkts/184 bytes -> 0 pkts/0 bytes][Goodput ratio: 54/0][1.02 sec][Host: nasfile][PLAIN TEXT ( EOEBFDEGEJEMEFCACACACACACACACA)]
-	63	UDP [fe80::e98f:bae2:19f7:6b0f]:51451 -> [ff02::1:3]:5355 [proto: 154/LLMNR][cat: Network/14][2 pkts/184 bytes -> 0 pkts/0 bytes][Goodput ratio: 32/0][0.41 sec][Host: ____________][Risk: ** Suspicious DGA domain name **]
-	64	UDP [fe80::e98f:bae2:19f7:6b0f]:54888 -> [ff02::1:3]:5355 [proto: 154/LLMNR][cat: Network/14][2 pkts/184 bytes -> 0 pkts/0 bytes][Goodput ratio: 32/0][0.10 sec][Host: ____________][Risk: ** Suspicious DGA domain name **]
-	65	UDP [fe80::e98f:bae2:19f7:6b0f]:58779 -> [ff02::1:3]:5355 [proto: 154/LLMNR][cat: Network/14][2 pkts/184 bytes -> 0 pkts/0 bytes][Goodput ratio: 32/0][0.41 sec][Host: ____________][Risk: ** Suspicious DGA domain name **]
+	63	UDP [fe80::e98f:bae2:19f7:6b0f]:51451 -> [ff02::1:3]:5355 [proto: 154/LLMNR][cat: Network/14][2 pkts/184 bytes -> 0 pkts/0 bytes][Goodput ratio: 32/0][0.41 sec][Host: ____________]
+	64	UDP [fe80::e98f:bae2:19f7:6b0f]:54888 -> [ff02::1:3]:5355 [proto: 154/LLMNR][cat: Network/14][2 pkts/184 bytes -> 0 pkts/0 bytes][Goodput ratio: 32/0][0.10 sec][Host: ____________]
+	65	UDP [fe80::e98f:bae2:19f7:6b0f]:58779 -> [ff02::1:3]:5355 [proto: 154/LLMNR][cat: Network/14][2 pkts/184 bytes -> 0 pkts/0 bytes][Goodput ratio: 32/0][0.41 sec][Host: ____________]
 	66	UDP [fe80::e034:7be:d8f9:6197]:49766 -> [ff02::1:3]:5355 [proto: 154/LLMNR][cat: Network/14][2 pkts/182 bytes -> 0 pkts/0 bytes][Goodput ratio: 32/0][0.41 sec][Host: charming-pc][PLAIN TEXT (charming)]
 	67	UDP [fe80::5d92:62a8:ebde:1319]:49735 -> [ff02::1:3]:5355 [proto: 154/LLMNR][cat: Network/14][2 pkts/178 bytes -> 0 pkts/0 bytes][Goodput ratio: 30/0][0.10 sec][Host: wangs-ltw]
 	68	UDP [fe80::5d92:62a8:ebde:1319]:58468 -> [ff02::1:3]:5355 [proto: 154/LLMNR][cat: Network/14][2 pkts/178 bytes -> 0 pkts/0 bytes][Goodput ratio: 30/0][0.10 sec][Host: wangs-ltw]
@@ -98,9 +98,9 @@ JA3 Host Stats:
 	76	UDP [fe80::edf5:240a:c8c0:8312]:61603 -> [ff02::1:3]:5355 [proto: 154/LLMNR][cat: Network/14][2 pkts/172 bytes -> 0 pkts/0 bytes][Goodput ratio: 28/0][0.41 sec][Host: ro_x1c]
 	77	UDP 192.168.5.49:61548 -> 224.0.0.252:5355 [proto: 154/LLMNR][cat: Network/14][2 pkts/150 bytes -> 0 pkts/0 bytes][Goodput ratio: 44/0][0.41 sec][Host: caesar-thinkpad][PLAIN TEXT (caesar)]
 	78	UDP 192.168.5.49:64568 -> 224.0.0.252:5355 [proto: 154/LLMNR][cat: Network/14][2 pkts/150 bytes -> 0 pkts/0 bytes][Goodput ratio: 44/0][0.41 sec][Host: caesar-thinkpad][PLAIN TEXT (caesar)]
-	79	UDP 192.168.3.95:51451 -> 224.0.0.252:5355 [proto: 154/LLMNR][cat: Network/14][2 pkts/144 bytes -> 0 pkts/0 bytes][Goodput ratio: 41/0][0.41 sec][Host: ____________][Risk: ** Suspicious DGA domain name **]
-	80	UDP 192.168.3.95:54888 -> 224.0.0.252:5355 [proto: 154/LLMNR][cat: Network/14][2 pkts/144 bytes -> 0 pkts/0 bytes][Goodput ratio: 41/0][0.10 sec][Host: ____________][Risk: ** Suspicious DGA domain name **]
-	81	UDP 192.168.3.95:58779 -> 224.0.0.252:5355 [proto: 154/LLMNR][cat: Network/14][2 pkts/144 bytes -> 0 pkts/0 bytes][Goodput ratio: 41/0][0.41 sec][Host: ____________][Risk: ** Suspicious DGA domain name **]
+	79	UDP 192.168.3.95:51451 -> 224.0.0.252:5355 [proto: 154/LLMNR][cat: Network/14][2 pkts/144 bytes -> 0 pkts/0 bytes][Goodput ratio: 41/0][0.41 sec][Host: ____________]
+	80	UDP 192.168.3.95:54888 -> 224.0.0.252:5355 [proto: 154/LLMNR][cat: Network/14][2 pkts/144 bytes -> 0 pkts/0 bytes][Goodput ratio: 41/0][0.10 sec][Host: ____________]
+	81	UDP 192.168.3.95:58779 -> 224.0.0.252:5355 [proto: 154/LLMNR][cat: Network/14][2 pkts/144 bytes -> 0 pkts/0 bytes][Goodput ratio: 41/0][0.41 sec][Host: ____________]
 	82	UDP 192.168.5.50:57143 -> 224.0.0.252:5355 [proto: 154/LLMNR][cat: Network/14][2 pkts/142 bytes -> 0 pkts/0 bytes][Goodput ratio: 41/0][0.41 sec][Host: charming-pc][PLAIN TEXT (charming)]
 	83	UDP 192.168.5.50:62756 -> 224.0.0.252:5355 [proto: 154/LLMNR][cat: Network/14][2 pkts/142 bytes -> 0 pkts/0 bytes][Goodput ratio: 41/0][0.41 sec][Host: charming-pc][PLAIN TEXT (charming)]
 	84	UDP 192.168.5.48:59797 -> 224.0.0.252:5355 [proto: 154/LLMNR][cat: Network/14][2 pkts/140 bytes -> 0 pkts/0 bytes][Goodput ratio: 40/0][0.41 sec][Host: kasper-mac][PLAIN TEXT (Kasper)]
diff --git a/tests/result/weibo.pcap.out b/tests/result/weibo.pcap.out
index e11aa8ed3..f90ce31de 100644
--- a/tests/result/weibo.pcap.out
+++ b/tests/result/weibo.pcap.out
@@ -28,7 +28,7 @@ JA3 Host Stats:
 	16	UDP 192.168.1.105:51440 <-> 192.168.1.1:53 [proto: 5/DNS][cat: Network/14][1 pkts/72 bytes <-> 1 pkts/171 bytes][Goodput ratio: 41/75][0.19 sec][Host: g.alicdn.com][47.89.65.229][PLAIN TEXT (alicdn)]
 	17	UDP 192.168.1.105:33822 <-> 192.168.1.1:53 [proto: 5/DNS][cat: Network/14][1 pkts/76 bytes <-> 1 pkts/166 bytes][Goodput ratio: 44/74][0.47 sec][Host: login.taobao.com][140.205.170.63][PLAIN TEXT (taobao)]
 	18	UDP 192.168.1.105:18035 <-> 192.168.1.1:53 [proto: 5.200/DNS.Sina(Weibo)][cat: SocialNetwork/6][1 pkts/81 bytes <-> 1 pkts/159 bytes][Goodput ratio: 48/73][0.11 sec][Host: u1.img.mobile.sina.cn][222.73.28.96][PLAIN TEXT (mobile)]
-	19	UDP 192.168.1.105:50640 <-> 192.168.1.1:53 [proto: 5/DNS][cat: Network/14][1 pkts/77 bytes <-> 1 pkts/157 bytes][Goodput ratio: 45/73][0.47 sec][Host: acjstb.aliyun.com][42.156.184.19][Risk: ** Suspicious DGA domain name **][PLAIN TEXT (alibabadns)]
+	19	UDP 192.168.1.105:50640 <-> 192.168.1.1:53 [proto: 5/DNS][cat: Network/14][1 pkts/77 bytes <-> 1 pkts/157 bytes][Goodput ratio: 45/73][0.47 sec][Host: acjstb.aliyun.com][42.156.184.19][PLAIN TEXT (alibabadns)]
 	20	UDP 192.168.1.105:7148 <-> 192.168.1.1:53 [proto: 5.200/DNS.Sina(Weibo)][cat: SocialNetwork/6][1 pkts/73 bytes <-> 1 pkts/142 bytes][Goodput ratio: 42/70][0.06 sec][Host: www.weibo.com][93.188.134.137]
 	21	TCP 192.168.1.105:35808 <-> 93.188.134.246:80 [proto: 7/HTTP][cat: Web/5][2 pkts/140 bytes <-> 1 pkts/74 bytes][Goodput ratio: 0/0][0.06 sec]
 	22	TCP 192.168.1.105:50831 <-> 47.89.65.229:443 [proto: 91/TLS][cat: Web/5][2 pkts/128 bytes <-> 1 pkts/66 bytes][Goodput ratio: 0/0][0.22 sec]