Improved detection of Android connectiity checks

4 files changed, 53 insertions, 1 deletions

diff --git a/src/lib/protocols/http.c b/src/lib/protocols/http.c
index c73bd47fe..bfb47f514 100644
--- a/src/lib/protocols/http.c
+++ b/src/lib/protocols/http.c
@@ -550,6 +550,16 @@ static void ndpi_http_parse_subprotocol(struct ndpi_detection_module_struct *ndp
     }
   }
 
+  if(flow->http.url
+     && (
+       ends_with(ndpi_struct, (char*)flow->http.url, "/generate_204")
+       || ends_with(ndpi_struct, (char*)flow->http.url, "/generate204")
+       )
+    ) {
+    flow->category = NDPI_PROTOCOL_CATEGORY_CONNECTIVITY_CHECK;
+    return;
+  }
+
   if(flow->detected_protocol_stack[1] == NDPI_PROTOCOL_UNKNOWN &&
      flow->http.url &&
      ((strstr(flow->http.url, ":8080/downloading?n=0.") != NULL) ||
diff --git a/tests/cfgs/default/pcap/conncheck.pcap b/tests/cfgs/default/pcap/conncheck.pcap
new file mode 100644
index 000000000..3aaa984fc
--- /dev/null
+++ b/tests/cfgs/default/pcap/conncheck.pcap
diff --git a/tests/cfgs/default/result/conncheck.pcap.out b/tests/cfgs/default/result/conncheck.pcap.out
new file mode 100644
index 000000000..fecd7efbd
--- /dev/null
+++ b/tests/cfgs/default/result/conncheck.pcap.out
@@ -0,0 +1,42 @@
+DPI Packets (TCP):	65	(7.22 pkts/flow)
+DPI Packets (UDP):	2	(2.00 pkts/flow)
+Confidence DPI              : 10 (flows)
+Num dissector calls: 136 (13.60 diss/flow)
+LRU cache ookla:      0/0/0 (insert/search/found)
+LRU cache bittorrent: 0/0/0 (insert/search/found)
+LRU cache stun:       0/0/0 (insert/search/found)
+LRU cache tls_cert:   0/0/0 (insert/search/found)
+LRU cache mining:     0/0/0 (insert/search/found)
+LRU cache msteams:    0/0/0 (insert/search/found)
+LRU cache stun_zoom:  0/0/0 (insert/search/found)
+Automa host:          11/6 (search/found)
+Automa domain:        11/0 (search/found)
+Automa tls cert:      0/0 (search/found)
+Automa risk mask:     1/0 (search/found)
+Automa common alpns:  0/0 (search/found)
+Patricia risk mask:   2/0 (search/found)
+Patricia risk mask IPv6: 0/0 (search/found)
+Patricia risk:        0/0 (search/found)
+Patricia risk IPv6:   0/0 (search/found)
+Patricia protocols:   14/6 (search/found)
+Patricia protocols IPv6: 0/0 (search/found)
+
+DNS	2	305	1
+HTTP	10	2272	1
+ntop	61	10472	6
+Google	10	2153	1
+GoogleServices	9	1912	1
+
+Safe                            61 10472         6            
+Acceptable                      31 6642          4            
+
+	1	TCP 10.1.0.60:38024 <-> 92.123.101.121:80 [proto: 7/HTTP][IP: 0/Unknown][ClearText][Confidence: DPI][DPI packets: 10][cat: ConnCheck/30][9 pkts/2198 bytes <-> 1 pkts/74 bytes][Goodput ratio: 73/0][7.08 sec][Hostname/SNI: conn-service-eu-04.allawnos.com][bytes ratio: 0.935 (Upload)][IAT c2s/s2c min/avg/max/stddev: 0/0 885/0 3618/0 1162/0][Pkt Len c2s/s2c min/avg/max/stddev: 66/74 244/74 294/74 93/0][URL: conn-service-eu-04.allawnos.com/generate204][User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.32 Safari/537.36][Risk: ** Probing attempt **][Risk Score: 50][Risk Info: TCP connection with unidirectional traffic][PLAIN TEXT (nGET /generate204 HTTP/1.1)][Plen Bins: 0,0,0,0,0,0,0,100,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
+	2	TCP 10.1.0.60:49674 <-> 142.250.180.163:80 [proto: 7.126/HTTP.Google][IP: 126/Google][ClearText][Confidence: DPI][DPI packets: 10][cat: ConnCheck/30][9 pkts/2079 bytes <-> 1 pkts/74 bytes][Goodput ratio: 71/0][7.05 sec][Hostname/SNI: www.google.eu][bytes ratio: 0.931 (Upload)][IAT c2s/s2c min/avg/max/stddev: 0/0 881/0 3584/0 1153/0][Pkt Len c2s/s2c min/avg/max/stddev: 66/74 231/74 277/74 86/0][URL: www.google.eu/generate_204][User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.32 Safari/537.36][Risk: ** Probing attempt **][Risk Score: 50][Risk Info: TCP connection with unidirectional traffic][PLAIN TEXT (GET /generate)][Plen Bins: 0,0,0,0,0,0,100,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
+	3	TCP 10.1.0.70:54612 <-> 142.250.180.138:80 [proto: 7.239/HTTP.GoogleServices][IP: 126/Google][ClearText][Confidence: DPI][DPI packets: 9][cat: ConnCheck/30][8 pkts/1838 bytes <-> 1 pkts/74 bytes][Goodput ratio: 71/0][3.67 sec][Hostname/SNI: play.googleapis.com][bytes ratio: 0.923 (Upload)][IAT c2s/s2c min/avg/max/stddev: 0/0 524/0 1824/0 607/0][Pkt Len c2s/s2c min/avg/max/stddev: 66/74 230/74 283/74 92/0][URL: play.googleapis.com/generate_204][User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.32 Safari/537.36][Risk: ** Probing attempt **][Risk Score: 50][Risk Info: TCP connection with unidirectional traffic][PLAIN TEXT (GET /generate)][Plen Bins: 0,0,0,0,0,0,100,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
+	4	TCP 10.1.0.60:46980 <-> 92.123.101.153:80 [proto: 7.26/HTTP.ntop][IP: 0/Unknown][ClearText][Confidence: DPI][DPI packets: 6][cat: ConnCheck/30][6 pkts/632 bytes <-> 5 pkts/1191 bytes][Goodput ratio: 36/72][0.01 sec][Hostname/SNI: conn-service-eu-04.allawnos.com][bytes ratio: -0.307 (Download)][IAT c2s/s2c min/avg/max/stddev: 0/0 3/4 5/7 2/3][Pkt Len c2s/s2c min/avg/max/stddev: 66/66 105/238 294/919 84/340][URL: conn-service-eu-04.allawnos.com/generate204][StatusCode: 302][Content-Type: text/html][Server: ntopng 6.1.240606 (x86_64)][User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.32 Safari/537.36][PLAIN TEXT (GET /generate)][Plen Bins: 0,0,0,0,0,0,0,50,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,50,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
+	5	TCP 10.1.0.60:38008 <-> 92.123.101.121:80 [proto: 7.26/HTTP.ntop][IP: 0/Unknown][ClearText][Confidence: DPI][DPI packets: 6][cat: ConnCheck/30][5 pkts/566 bytes <-> 5 pkts/1191 bytes][Goodput ratio: 40/72][0.03 sec][Hostname/SNI: conn-service-eu-04.allawnos.com][bytes ratio: -0.356 (Download)][IAT c2s/s2c min/avg/max/stddev: 0/0 7/10 21/21 9/9][Pkt Len c2s/s2c min/avg/max/stddev: 66/66 113/238 294/919 90/340][URL: conn-service-eu-04.allawnos.com/generate204][StatusCode: 302][Content-Type: text/html][Server: ntopng 6.1.240606 (x86_64)][User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.32 Safari/537.36][PLAIN TEXT (GET /generate)][Plen Bins: 0,0,0,0,0,0,0,50,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,50,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
+	6	TCP 10.1.0.60:49642 <-> 142.250.180.163:80 [proto: 7.26/HTTP.ntop][IP: 126/Google][ClearText][Confidence: DPI][DPI packets: 6][cat: ConnCheck/30][5 pkts/549 bytes <-> 5 pkts/1174 bytes][Goodput ratio: 38/71][0.02 sec][Hostname/SNI: www.google.eu][bytes ratio: -0.363 (Download)][IAT c2s/s2c min/avg/max/stddev: 0/0 3/5 6/7 3/3][Pkt Len c2s/s2c min/avg/max/stddev: 66/66 110/235 277/902 84/334][URL: www.google.eu/generate_204][StatusCode: 302][Content-Type: text/html][Server: ntopng 6.1.240606 (x86_64)][User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.32 Safari/537.36][PLAIN TEXT (GET /generate)][Plen Bins: 0,0,0,0,0,0,50,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,50,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
+	7	TCP 10.1.0.60:49656 <-> 142.250.180.163:80 [proto: 7.26/HTTP.ntop][IP: 126/Google][ClearText][Confidence: DPI][DPI packets: 6][cat: ConnCheck/30][5 pkts/549 bytes <-> 5 pkts/1174 bytes][Goodput ratio: 38/71][0.01 sec][Hostname/SNI: www.google.eu][bytes ratio: -0.363 (Download)][IAT c2s/s2c min/avg/max/stddev: 0/0 3/4 7/7 3/3][Pkt Len c2s/s2c min/avg/max/stddev: 66/66 110/235 277/902 84/334][URL: www.google.eu/generate_204][StatusCode: 302][Content-Type: text/html][Server: ntopng 6.1.240606 (x86_64)][User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.32 Safari/537.36][PLAIN TEXT (GET /generate)][Plen Bins: 0,0,0,0,0,0,50,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,50,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
+	8	TCP 10.1.0.60:49658 <-> 142.250.180.163:80 [proto: 7.26/HTTP.ntop][IP: 126/Google][ClearText][Confidence: DPI][DPI packets: 6][cat: ConnCheck/30][5 pkts/549 bytes <-> 5 pkts/1174 bytes][Goodput ratio: 38/71][0.03 sec][Hostname/SNI: www.google.eu][bytes ratio: -0.363 (Download)][IAT c2s/s2c min/avg/max/stddev: 0/0 8/10 15/21 5/9][Pkt Len c2s/s2c min/avg/max/stddev: 66/66 110/235 277/902 84/334][URL: www.google.eu/generate_204][StatusCode: 302][Content-Type: text/html][Server: ntopng 6.1.240606 (x86_64)][User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.32 Safari/537.36][PLAIN TEXT (GET /generate)][Plen Bins: 0,0,0,0,0,0,50,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,50,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
+	9	TCP 10.1.0.60:49672 <-> 142.250.180.163:80 [proto: 7.26/HTTP.ntop][IP: 126/Google][ClearText][Confidence: DPI][DPI packets: 6][cat: ConnCheck/30][5 pkts/549 bytes <-> 5 pkts/1174 bytes][Goodput ratio: 38/71][0.01 sec][Hostname/SNI: www.google.eu][bytes ratio: -0.363 (Download)][IAT c2s/s2c min/avg/max/stddev: 1/0 2/3 5/7 2/3][Pkt Len c2s/s2c min/avg/max/stddev: 66/66 110/235 277/902 84/334][URL: www.google.eu/generate_204][StatusCode: 302][Content-Type: text/html][Server: ntopng 6.1.240606 (x86_64)][User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.32 Safari/537.36][PLAIN TEXT (GET /generate)][Plen Bins: 0,0,0,0,0,0,50,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,50,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
+	10	UDP 10.1.0.60:46571 <-> 10.1.0.1:53 [proto: 5/DNS][IP: 0/Unknown][ClearText][Confidence: DPI][DPI packets: 2][cat: Network/14][1 pkts/91 bytes <-> 1 pkts/214 bytes][Goodput ratio: 53/80][0.01 sec][Hostname/SNI: conn-service-eu-04.allawnos.com][92.123.101.121][PLAIN TEXT (service)][Plen Bins: 0,50,0,0,0,50,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
diff --git a/tests/cfgs/default/result/quickplay.pcap.out b/tests/cfgs/default/result/quickplay.pcap.out
index d960e80d1..291a359cb 100644
--- a/tests/cfgs/default/result/quickplay.pcap.out
+++ b/tests/cfgs/default/result/quickplay.pcap.out
@@ -49,4 +49,4 @@ Fun                             18 6521          8
 	18	TCP 10.54.169.250:52288 <-> 173.252.74.22:80 [proto: 7.119/HTTP.Facebook][IP: 119/Facebook][ClearText][Confidence: DPI][DPI packets: 2][cat: SocialNetwork/6][1 pkts/243 bytes <-> 1 pkts/339 bytes][Goodput ratio: 77/83][0.46 sec][Hostname/SNI: www.facebook.com][URL: www.facebook.com/mobile/status.php][StatusCode: 204][User-Agent: Dalvik/1.6.0 (Linux; U; Android 4.4.4; MI 3W MIUI/V6.4.2.0.KXDMICB)][PLAIN TEXT (GET /mobile/status.php HTTP/1.1)][Plen Bins: 0,0,0,0,0,50,0,0,50,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
 	19	TCP 10.54.169.250:44793 <-> 31.13.68.49:80 [proto: 7.119/HTTP.Facebook][IP: 119/Facebook][ClearText][Confidence: DPI][DPI packets: 2][cat: SocialNetwork/6][1 pkts/237 bytes <-> 1 pkts/339 bytes][Goodput ratio: 76/83][0.34 sec][Hostname/SNI: www.facebook.com][URL: www.facebook.com/mobile/status.php][StatusCode: 204][User-Agent: Dalvik/1.6.0 (Linux; U; Android 4.4.2; GT-I9505 Build/KOT49H)][PLAIN TEXT (GET /mobile/status.php HTTP/1.1)][Plen Bins: 0,0,0,0,0,50,0,0,50,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
 	20	TCP 10.54.169.250:33064 <-> 120.28.5.18:80 [proto: 7/HTTP][IP: 0/Unknown][ClearText][Confidence: DPI][DPI packets: 2][cat: Streaming/17][1 pkts/358 bytes <-> 1 pkts/109 bytes][Goodput ratio: 84/48][0.31 sec][Hostname/SNI: api-singtelhawk.quickplay.com][URL: api-singtelhawk.quickplay.com/solr/RestApiSingTel_PH/restapi/home?apiKey=qwerty&device=androidmobile&locale=eng&network=WIFI&pageNumber=1&pageSize=50][User-Agent: Dalvik/1.6.0 (Linux; U; Android 4.4.4; MI 3W MIUI/V6.4.2.0.KXDMICB)][PLAIN TEXT (GET /solr/RestApiSingTel)][Plen Bins: 0,50,0,0,0,0,0,0,0,50,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
-	21	TCP 10.54.169.250:33277 <-> 120.28.26.231:80 [proto: 7.126/HTTP.Google][IP: 0/Unknown][ClearText][Confidence: DPI][DPI packets: 2][cat: Web/5][1 pkts/241 bytes <-> 1 pkts/137 bytes][Goodput ratio: 76/59][0.06 sec][Hostname/SNI: clients3.google.com][URL: clients3.google.com/generate_204][StatusCode: 204][Server: GFE/2.0][User-Agent: Dalvik/1.6.0 (Linux; U; Android 4.4.4; MI 3W MIUI/V6.4.2.0.KXDMICB)][Risk: ** Susp Entropy **][Risk Score: 10][Risk Info: Entropy: 5.305 (Executable?)][PLAIN TEXT (GET /generate)][Plen Bins: 0,0,50,0,0,50,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
+	21	TCP 10.54.169.250:33277 <-> 120.28.26.231:80 [proto: 7.126/HTTP.Google][IP: 0/Unknown][ClearText][Confidence: DPI][DPI packets: 2][cat: ConnCheck/30][1 pkts/241 bytes <-> 1 pkts/137 bytes][Goodput ratio: 76/59][0.06 sec][Hostname/SNI: clients3.google.com][URL: clients3.google.com/generate_204][StatusCode: 204][Server: GFE/2.0][User-Agent: Dalvik/1.6.0 (Linux; U; Android 4.4.4; MI 3W MIUI/V6.4.2.0.KXDMICB)][PLAIN TEXT (GET /generate)][Plen Bins: 0,0,50,0,0,50,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]