Added newflow risk NDPI_HTTP_CRAWLER_BOT

4 files changed, 42 insertions, 14 deletions

diff --git a/src/include/ndpi_typedefs.h b/src/include/ndpi_typedefs.h
index 0b798c530..a86fa79da 100644
--- a/src/include/ndpi_typedefs.h
+++ b/src/include/ndpi_typedefs.h
@@ -116,6 +116,7 @@ typedef enum {
   NDPI_TLS_CERTIFICATE_ABOUT_TO_EXPIRE,
   NDPI_PUNYCODE_IDN, /* https://en.wikipedia.org/wiki/Punycode */
   NDPI_ERROR_CODE_DETECTED,
+  NDPI_HTTP_CRAWLER_BOT,
   
   /* Leave this as last member */
   NDPI_MAX_RISK /* must be <= 63 due to (**) */
diff --git a/src/lib/ndpi_main.c b/src/lib/ndpi_main.c
index 7c0e8f3b1..2740ec8bb 100644
--- a/src/lib/ndpi_main.c
+++ b/src/lib/ndpi_main.c
@@ -125,6 +125,7 @@ static ndpi_risk_info ndpi_known_risks[] = {
   { NDPI_TLS_CERTIFICATE_ABOUT_TO_EXPIRE,       NDPI_RISK_MEDIUM, CLIENT_LOW_RISK_PERCENTAGE  },
   { NDPI_PUNYCODE_IDN,                          NDPI_RISK_LOW,    CLIENT_LOW_RISK_PERCENTAGE  },
   { NDPI_ERROR_CODE_DETECTED,                   NDPI_RISK_LOW,    CLIENT_LOW_RISK_PERCENTAGE  },
+  { NDPI_HTTP_CRAWLER_BOT,                      NDPI_RISK_LOW,    CLIENT_LOW_RISK_PERCENTAGE  },
   
   /* Leave this as last member */
   { NDPI_MAX_RISK,                              NDPI_RISK_LOW,    CLIENT_FAIR_RISK_PERCENTAGE }
diff --git a/src/lib/ndpi_utils.c b/src/lib/ndpi_utils.c
index fabc4db2a..29cb94695 100644
--- a/src/lib/ndpi_utils.c
+++ b/src/lib/ndpi_utils.c
@@ -1852,6 +1852,10 @@ const char* ndpi_risk2str(ndpi_risk_enum risk) {
     return("Error Code Detected");
     break;
       
+  case NDPI_HTTP_CRAWLER_BOT:
+    return("Crawler/Bot Detected");
+    break;
+      
   default:
     snprintf(buf, sizeof(buf), "%d", (int)risk);
     return(buf);
diff --git a/src/lib/protocols/http.c b/src/lib/protocols/http.c
index cf1e6282b..b34206271 100644
--- a/src/lib/protocols/http.c
+++ b/src/lib/protocols/http.c
@@ -422,31 +422,53 @@ static void ndpi_check_user_agent(struct ndpi_detection_module_struct *ndpi_stru
 				  struct ndpi_flow_struct *flow,
 				  char *ua) {
   u_int len;
-
+  char *double_slash;
+  
   if((!ua) || (ua[0] == '\0'))
     return;
   else
     len = strlen(ua);
 
-  if(
-     (!strncmp(ua, "<?", 2))
-	  || strchr(ua, '$')
-	  || strstr(ua, "://") // || (!strncmp(ua, "jndi:ldap://", 12)) /* Log4J */
-	  // || ndpi_check_dga_name(ndpi_struct, NULL, ua, 0)
-	  // || ndpi_match_bigram(ndpi_struct, &ndpi_struct->impossible_bigrams_automa, ua)
-	  ) {
+  if((!strncmp(ua, "<?", 2))
+     || strchr(ua, '$')
+     // || ndpi_check_dga_name(ndpi_struct, NULL, ua, 0)
+     // || ndpi_match_bigram(ndpi_struct, &ndpi_struct->impossible_bigrams_automa, ua)
+     )
     ndpi_set_risk(ndpi_struct, flow, NDPI_HTTP_SUSPICIOUS_USER_AGENT);
 
+  if((double_slash = strstr(ua, "://")) != NULL) {
+    if(double_slash != ua) /* We're not at the beginning of the user agent */{
+      if((double_slash[-1] != 'p') /* http:// */
+	 && (double_slash[-1] != 's') /* https:// */)
+	ndpi_set_risk(ndpi_struct, flow, NDPI_HTTP_SUSPICIOUS_USER_AGENT);
+    }
+  }
+  
+  /* no else */
+  if(!strncmp(ua, "jndi:ldap://", 12)) /* Log4J */ {
     ndpi_set_risk(ndpi_struct, flow, NDPI_POSSIBLE_EXPLOIT);
   } else if(
-	    (len < 4)      /* Too short */
-	    || (len > 256) /* Too long  */
-	    || (!strncmp(ua, "test", 4))
-	    || strchr(ua, '{')
-	    || strchr(ua, '}')
-	    ) {
+	  (len < 4)      /* Too short */
+	  || (len > 256) /* Too long  */
+	  || (!strncmp(ua, "test", 4))
+	  || strchr(ua, '{')
+	  || strchr(ua, '}')
+	  ) {
     ndpi_set_risk(ndpi_struct, flow, NDPI_HTTP_SUSPICIOUS_USER_AGENT);
   }
+
+  /*
+    Mozilla/5.0 (compatible; YandexBot/3.0; +http://yandex.com/bots)
+    Amazon-Route53-Health-Check-Service (ref 68784dad-be98-49e4-a63c-9fbbe2816d7c; report http://amzn.to/1vsZADi)
+    Anonymous Crawler/1.0 (Webcrawler developed with StormCrawler; http://example.com/; webcrawler@example.com)
+   */
+  if((strstr(ua, "+http") != NULL)
+     || (strstr(ua, " http") != NULL)
+     || strcasestr(ua, "Crawler")
+     || strcasestr(ua, "Bot") /* bot/robot */
+     ) {
+    ndpi_set_risk(ndpi_struct, flow, NDPI_HTTP_CRAWLER_BOT);
+  }
 }
 
 /* ************************************************************* */