Added HTTP suspicious content securirty risk (useful for tracking trickbot)

7 files changed, 95 insertions, 21 deletions

diff --git a/python/ndpi.py b/python/ndpi.py
index 066ca2e9f..6bc166487 100644
--- a/python/ndpi.py
+++ b/python/ndpi.py
@@ -316,6 +316,7 @@ typedef enum {
   NDPI_UNSAFE_PROTOCOL,
   NDPI_DNS_SUSPICIOUS_TRAFFIC,
   NDPI_TLS_MISSING_SNI,
+  NDPI_HTTP_SUSPICIOUS_CONTENT,
   /* Leave this as last member */
   NDPI_MAX_RISK
 } ndpi_risk_enum;
diff --git a/src/include/ndpi_typedefs.h b/src/include/ndpi_typedefs.h
index 41938ebf5..46a7062c4 100644
--- a/src/include/ndpi_typedefs.h
+++ b/src/include/ndpi_typedefs.h
@@ -83,6 +83,7 @@ typedef enum {
   NDPI_UNSAFE_PROTOCOL,
   NDPI_DNS_SUSPICIOUS_TRAFFIC,
   NDPI_TLS_MISSING_SNI,
+  NDPI_HTTP_SUSPICIOUS_CONTENT,
   
   /* Leave this as last member */
   NDPI_MAX_RISK /* must be <= 31 due to (**) */
diff --git a/src/lib/ndpi_utils.c b/src/lib/ndpi_utils.c
index 95a115110..97b94ed68 100644
--- a/src/lib/ndpi_utils.c
+++ b/src/lib/ndpi_utils.c
@@ -1721,6 +1721,9 @@ const char* ndpi_risk2str(ndpi_risk_enum risk) {
   case NDPI_TLS_MISSING_SNI:
     return("SNI TLS extension was missing");
     
+  case NDPI_HTTP_SUSPICIOUS_CONTENT:
+    return("HTTP suspicious content");
+    
   default:
     snprintf(buf, sizeof(buf), "%d", (int)risk);
     return(buf);
diff --git a/src/lib/protocols/http.c b/src/lib/protocols/http.c
index f130ea6b9..70750fbe0 100644
--- a/src/lib/protocols/http.c
+++ b/src/lib/protocols/http.c
@@ -423,28 +423,28 @@ static void check_content_type_and_change_protocol(struct ndpi_detection_module_
     return;
 
   if((flow->http.url == NULL)
-       && (packet->http_url_name.len > 0)
-       && (packet->host_line.len > 0)) {
-      int len = packet->http_url_name.len + packet->host_line.len + 1;
-
-      if(isdigit(packet->host_line.ptr[0])
-	 && (packet->host_line.len < 21))
-	ndpi_check_numeric_ip(ndpi_struct, flow, (char*)packet->host_line.ptr, packet->host_line.len);
-
-      flow->http.url = ndpi_malloc(len);
-      if(flow->http.url) {
-	strncpy(flow->http.url, (char*)packet->host_line.ptr, packet->host_line.len);
-	strncpy(&flow->http.url[packet->host_line.len], (char*)packet->http_url_name.ptr,
-		packet->http_url_name.len);
-	flow->http.url[len-1] = '\0';
-
-	ndpi_check_http_url(ndpi_struct, flow, &flow->http.url[packet->host_line.len]);
-      }
-
-      flow->http.method = ndpi_http_str2method((const char*)flow->packet.http_method.ptr,
-					       (u_int16_t)flow->packet.http_method.len);
+     && (packet->http_url_name.len > 0)
+     && (packet->host_line.len > 0)) {
+    int len = packet->http_url_name.len + packet->host_line.len + 1;
+
+    if(isdigit(packet->host_line.ptr[0])
+       && (packet->host_line.len < 21))
+      ndpi_check_numeric_ip(ndpi_struct, flow, (char*)packet->host_line.ptr, packet->host_line.len);
+
+    flow->http.url = ndpi_malloc(len);
+    if(flow->http.url) {
+      strncpy(flow->http.url, (char*)packet->host_line.ptr, packet->host_line.len);
+      strncpy(&flow->http.url[packet->host_line.len], (char*)packet->http_url_name.ptr,
+	      packet->http_url_name.len);
+      flow->http.url[len-1] = '\0';
+
+      ndpi_check_http_url(ndpi_struct, flow, &flow->http.url[packet->host_line.len]);
     }
 
+    flow->http.method = ndpi_http_str2method((const char*)flow->packet.http_method.ptr,
+					     (u_int16_t)flow->packet.http_method.len);
+  }
+
   if(packet->server_line.ptr != NULL && (packet->server_line.len > 7)) {
     if(strncmp((const char *)packet->server_line.ptr, "ntopng ", 7) == 0) {
       ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_NTOP, NDPI_PROTOCOL_HTTP);
@@ -749,6 +749,15 @@ static void ndpi_check_http_header(struct ndpi_detection_module_struct *ndpi_str
 
 /*************************************************************************************************/
 
+static int ndpi_http_is_print(char c) {
+  if(isprint(c) || (c == '\t') || (c == '\r') || (c == '\n'))
+    return(1);
+  else
+    return(0);
+}
+
+/*************************************************************************************************/
+
 static void ndpi_check_http_tcp(struct ndpi_detection_module_struct *ndpi_struct,
 				struct ndpi_flow_struct *flow) {
   struct ndpi_packet_struct *packet = &flow->packet;
@@ -787,6 +796,63 @@ static void ndpi_check_http_tcp(struct ndpi_detection_module_struct *ndpi_struct
 
 	ndpi_parse_packet_line_info(ndpi_struct, flow);
         check_content_type_and_change_protocol(ndpi_struct, flow);
+
+
+	{
+	  const u_int8_t *double_ret = (const u_int8_t *)ndpi_strnstr((const char *)packet->payload, "\r\n\r\n", packet->payload_packet_len);
+
+#ifdef NDPI_ENABLE_DEBUG_MESSAGES
+	  printf("==>>> [len: %u] ", packet->payload_packet_len);
+#endif
+	  
+	  if(double_ret) {
+	    u_int len;
+
+	    len = packet->payload_packet_len - (double_ret - packet->payload);
+
+	    if(len >= 8 /* 4 chars for \r\n\r\n and at least 4 charts for content guess */) {
+	      double_ret += 4;
+	      
+#ifdef NDPI_ENABLE_DEBUG_MESSAGES
+	      int i;
+	      
+	      for(i=0; i<packet->content_line.len; i++)
+		printf("%c", packet->content_line.ptr[i]);
+  
+	      printf(" [len: %u] [%02X %02X %02X %02X][%c%c%c%c]", len,
+		     double_ret[0], double_ret[1], double_ret[2], double_ret[3],
+		     double_ret[0], double_ret[1], double_ret[2], double_ret[3]
+		     );
+#endif
+	      
+	      if(strnstr((const char *)packet->content_line.ptr, "text/", packet->content_line.len)
+		 || strnstr((const char *)packet->content_line.ptr, "/json", packet->content_line.len)
+		 ) {
+		/* This is supposed to be a hunan-readeable text file */
+
+		if(ndpi_http_is_print(double_ret[0]) && ndpi_http_is_print(double_ret[1])
+		   && ndpi_http_is_print(double_ret[2]) && ndpi_http_is_print(double_ret[3])) {
+		  /* OK */
+		} else {
+		  /* Looks bad: last resort check if it's gzipped [1F 8B 08 00] */
+
+		  if((double_ret[0] == 0x1F)
+		     && (double_ret[1] == 0x8B)
+		     && (double_ret[2] == 0x08)
+		     && (double_ret[3] == 0x00)) {
+		    /* Looks like compressed data */
+		  } else
+		    NDPI_SET_BIT(flow->risk, NDPI_HTTP_SUSPICIOUS_CONTENT);		  
+		}
+	      }
+	    }
+
+#ifdef NDPI_ENABLE_DEBUG_MESSAGES
+	    printf("\n");
+#endif
+	  }
+	}
+	
         return;
       }
 
diff --git a/tests/pcap/trickbot.pcap b/tests/pcap/trickbot.pcap
new file mode 100644
index 000000000..7afef55fc
--- /dev/null
+++ b/tests/pcap/trickbot.pcap
diff --git a/tests/result/netflix.pcap.out b/tests/result/netflix.pcap.out
index ef4a63dd8..7f5a6e42e 100644
--- a/tests/result/netflix.pcap.out
+++ b/tests/result/netflix.pcap.out
@@ -12,7 +12,7 @@ JA3 Host Stats:
 	1	TCP 192.168.1.7:53217 <-> 23.246.11.141:80 [proto: 7.133/HTTP.NetFlix][cat: Video/26][667 pkts/50462 bytes <-> 1205 pkts/1807875 bytes][Goodput ratio: 12/96][26.40 sec][Host: 23.246.11.141][bytes ratio: -0.946 (Download)][IAT c2s/s2c min/avg/max/stddev: 0/0 33/21 522/505 51/40][Pkt Len c2s/s2c min/avg/max/stddev: 60/74 76/1500 584/1514 69/116][URL: 23.246.11.141/?o=AQEfKq2oMrLRiWL2puNQJJ2TLhuiGLjSseu23V2HX6kIiU9JpbCaBxxaIoz21qQNKuDUaOIZwdTlx23DMVxabbCwmvEluipDW2tvFMlhMRtwdhhVlbv9KGFabiu5KH0Slx0VjOK_wzThpP7lHhWA4kW9gayYEWtjNNKe&v=3&e=1484347850&t=Dh278u2UpApOCGUj5RxV8azNWX8][StatusCode: 206][User-Agent: AppleCoreMedia/1.0.0.14C92 (iPhone; U; CPU OS 10_2 like Mac OS X; en_us)][Risk: ** HTTP Numeric IP Address **][PLAIN TEXT (oMrLRiWL2)][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,100,0,0]
 	2	TCP 192.168.1.7:53183 <-> 23.246.3.140:80 [proto: 7.133/HTTP.NetFlix][cat: Video/26][502 pkts/40335 bytes <-> 805 pkts/1202445 bytes][Goodput ratio: 17/96][53.10 sec][Host: 23.246.3.140][bytes ratio: -0.935 (Download)][IAT c2s/s2c min/avg/max/stddev: 0/0 117/55 5026/5044 455/248][Pkt Len c2s/s2c min/avg/max/stddev: 60/74 80/1494 581/1514 81/140][URL: 23.246.3.140/?o=AQEfKq2oMrLRiWL-p-VeIZ6WKRq-X6LMvaLqgxWBCuFbh09MpreORUUOO5Tx1683HPnLY6BPjN_9mlDuYihGZoXu9u0ozH8RFioBN_JDNiRscidjvoSdWmlyZgPNansW0lkBr4X81HvloOi8BS_exVSPhMyJQTB5bg&v=3&e=1484347850&t=-8u4vlcPuFqcOLnLyb9DDtK-bB4][StatusCode: 206][User-Agent: AppleCoreMedia/1.0.0.14C92 (iPhone; U; CPU OS 10_2 like Mac OS X; en_us)][Risk: ** HTTP Numeric IP Address **][PLAIN TEXT (oMrLRiWL)][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,97,0,0]
 	3	TCP 192.168.1.7:53210 <-> 23.246.11.133:80 [proto: 7.133/HTTP.NetFlix][cat: Video/26][293 pkts/23170 bytes <-> 495 pkts/736113 bytes][Goodput ratio: 16/96][46.97 sec][Host: 23.246.11.133][bytes ratio: -0.939 (Download)][IAT c2s/s2c min/avg/max/stddev: 0/0 194/107 26359/26393 1829/1321][Pkt Len c2s/s2c min/avg/max/stddev: 66/66 79/1487 582/1514 79/167][URL: 23.246.11.133/?o=AQEfKq2oMrLRiWL1ouVaJpeQLBWjGLjSseu23V2HX6kIiU9JpbCaBxxaIoz21qQNKuDUaOIZwdTlx23DMVxabbCwmvEluipDW2tvFMlhMRtwdhhVlbv9KGFabiu5KH0Slx0VjOK_wzThp_7lHhWA4kW9gayYEWtjNNKe&v=3&e=1484347850&t=JfEef80K02ynIjLLoi-HZB1uQ10][StatusCode: 206][User-Agent: AppleCoreMedia/1.0.0.14C92 (iPhone; U; CPU OS 10_2 like Mac OS X; en_us)][Risk: ** HTTP Numeric IP Address **][PLAIN TEXT (oMrLRiWL1)][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,96,0,0]
-	4	TCP 192.168.1.7:53153 <-> 184.25.204.24:80 [proto: 7.133/HTTP.NetFlix][cat: Video/26][147 pkts/11558 bytes <-> 490 pkts/734346 bytes][Goodput ratio: 2/96][59.61 sec][Host: tp.akam.nflximg.com][bytes ratio: -0.969 (Download)][IAT c2s/s2c min/avg/max/stddev: 2/0 418/45 30607/2159 2956/164][Pkt Len c2s/s2c min/avg/max/stddev: 66/66 79/1499 282/1514 21/140][URL: tp.akam.nflximg.com/tpa3/616/2041779616.bif][StatusCode: 200][Content-Type: text/plain][User-Agent: Argo/900 CFNetwork/808.2.16 Darwin/16.3.0][PLAIN TEXT (GET /tpa3/616/2041779616.bif HT)][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,100,0,0]
+	4	TCP 192.168.1.7:53153 <-> 184.25.204.24:80 [proto: 7.133/HTTP.NetFlix][cat: Video/26][147 pkts/11558 bytes <-> 490 pkts/734346 bytes][Goodput ratio: 2/96][59.61 sec][Host: tp.akam.nflximg.com][bytes ratio: -0.969 (Download)][IAT c2s/s2c min/avg/max/stddev: 2/0 418/45 30607/2159 2956/164][Pkt Len c2s/s2c min/avg/max/stddev: 66/66 79/1499 282/1514 21/140][URL: tp.akam.nflximg.com/tpa3/616/2041779616.bif][StatusCode: 200][Content-Type: text/plain][User-Agent: Argo/900 CFNetwork/808.2.16 Darwin/16.3.0][Risk: ** HTTP suspicious content **][PLAIN TEXT (GET /tpa3/616/2041779616.bif HT)][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,100,0,0]
 	5	TCP 192.168.1.7:53141 <-> 104.86.97.179:443 [proto: 91.133/TLS.NetFlix][cat: Video/26][83 pkts/7225 bytes <-> 147 pkts/202723 bytes][Goodput ratio: 20/95][73.78 sec][ALPN: h2;h2-16;h2-15;h2-14;spdy/3.1;spdy/3;http/1.1][bytes ratio: -0.931 (Download)][IAT c2s/s2c min/avg/max/stddev: 0/0 1184/604 69170/69192 8780/6263][Pkt Len c2s/s2c min/avg/max/stddev: 66/54 87/1379 293/1514 39/401][TLSv1.2][Client: art-s.nflximg.net][JA3C: c07cb55f88702033a8f52c046d23e0b2][ServerNames: secure.cdn.nflximg.net,*.nflxext.com,*.nflxvideo.net,*.nflxsearch.net,*.nrd.nflximg.net,*.nflximg.net][JA3S: ef6b224ce027c8e21e5a25d8a58255a3][Issuer: C=US, O=Symantec Corporation, OU=Symantec Trust Network, CN=Symantec Class 3 Secure Server CA - G4][Subject: C=US, ST=California, L=Los Gatos, O=Netflix, Inc., OU=Content Delivery Operations, CN=secure.cdn.nflximg.net][Certificate SHA-1: 0D:EF:D1:E6:29:11:1A:A5:88:B3:2F:04:65:D6:D7:AD:84:A2:52:26][Validity: 2016-04-06 00:00:00 - 2017-04-05 23:59:59][Cipher: TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384][Plen Bins: 1,4,6,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,87,0,0]
 	6	TCP 192.168.1.7:53184 <-> 23.246.11.141:80 [proto: 7.133/HTTP.NetFlix][cat: Video/26][75 pkts/6610 bytes <-> 103 pkts/150772 bytes][Goodput ratio: 23/95][6.10 sec][Host: 23.246.11.141][bytes ratio: -0.916 (Download)][IAT c2s/s2c min/avg/max/stddev: 0/5 90/58 504/714 130/109][Pkt Len c2s/s2c min/avg/max/stddev: 60/74 88/1464 582/1514 100/228][URL: 23.246.11.141/?o=AQEfKq2oMrLRiWL2puNQJJqTIRqhGLjSseu23V2HX6kIiU9JpbCaBxxaIoz21qQNKuDUaOIZwdTlx23DMVxabbCwmvEluipDW2tvFMlhMRtwdhhVlbv9KGFabiu5KH0Slx0VjOK_wzThp_vlHhWA4kW9gayYEWtjNNKe&v=3&e=1484347850&t=TnP59JB1wb5UTOCr0m-KQU2kGPo][StatusCode: 206][User-Agent: AppleCoreMedia/1.0.0.14C92 (iPhone; U; CPU OS 10_2 like Mac OS X; en_us)][Risk: ** HTTP Numeric IP Address **][PLAIN TEXT (oMrLRiWL2)][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,98,0,0]
 	7	TCP 192.168.1.7:53149 <-> 184.25.204.25:80 [proto: 7.133/HTTP.NetFlix][cat: Video/26][40 pkts/3413 bytes <-> 86 pkts/125190 bytes][Goodput ratio: 7/95][34.92 sec][Host: art-2.nflximg.net][bytes ratio: -0.947 (Download)][IAT c2s/s2c min/avg/max/stddev: 6/12 1101/41 30978/402 5647/66][Pkt Len c2s/s2c min/avg/max/stddev: 66/66 85/1456 311/1514 38/274][URL: art-2.nflximg.net/5758c/bb636e44b87ef854c331ed7b7b6e157e4945758c.jpg][StatusCode: 200][Content-Type: image/jpeg][User-Agent: Argo/9.1.0 (iPhone; iOS 10.2; Scale/2.00)][PLAIN TEXT (GET /5758)][Plen Bins: 0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,97,0,0]
diff --git a/tests/result/trickbot.pcap.out b/tests/result/trickbot.pcap.out
new file mode 100644
index 000000000..4909d14d6
--- /dev/null
+++ b/tests/result/trickbot.pcap.out
@@ -0,0 +1,3 @@
+HTTP	74	62002	1
+
+	1	TCP 10.12.29.101:61318 <-> 82.118.225.196:7080 [proto: 7/HTTP][cat: Web/5][28 pkts/2801 bytes <-> 46 pkts/59201 bytes][Goodput ratio: 46/96][8.40 sec][Host: 82.118.225.196][bytes ratio: -0.910 (Download)][IAT c2s/s2c min/avg/max/stddev: 0/0 327/167 1000/1000 339/292][Pkt Len c2s/s2c min/avg/max/stddev: 54/54 100/1287 982/1514 182/426][URL: 82.118.225.196:7080/OK21pqJAtyyGBEo00sk][StatusCode: 200][Content-Type: application/x-www-form-urlencoded][User-Agent: Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 10.0; WOW64; Trident/7.0; .NET4.0C; .NET4.0E)][Risk: ** Known protocol on non standard port **** HTTP Numeric IP Address **** HTTP suspicious content **][PLAIN TEXT (POST /OK21p)][Plen Bins: 0,0,0,0,0,0,0,2,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,2,0,0,0,0,4,0,0,6,2,0,35,0,0,44,0,0]