Add Yandex services detection (#1882)

Add Yandex services detection

Add VK and Yandex to the TLS certificate match list

5 files changed, 106 insertions, 41 deletions

diff --git a/src/include/ndpi_protocol_ids.h b/src/include/ndpi_protocol_ids.h
index 926485c02..f0340f866 100644
--- a/src/include/ndpi_protocol_ids.h
+++ b/src/include/ndpi_protocol_ids.h
@@ -53,7 +53,7 @@ typedef enum {
   NDPI_PROTOCOL_VK                    = 22,
   NDPI_PROTOCOL_MAIL_POPS             = 23,
   NDPI_PROTOCOL_TAILSCALE             = 24,
-  NDPI_PROTOCOL_FREE_25               = 25, /* FREE */
+  NDPI_PROTOCOL_YANDEX                = 25,
   NDPI_PROTOCOL_NTOP                  = 26,
   NDPI_PROTOCOL_COAP                  = 27,
   NDPI_PROTOCOL_VMWARE                = 28,
@@ -61,8 +61,8 @@ typedef enum {
   NDPI_PROTOCOL_DTLS                  = 30,
   NDPI_PROTOCOL_UBNTAC2               = 31, /* Ubiquity UBNT AirControl = 2 */
   NDPI_PROTOCOL_KONTIKI               = 32,
-  NDPI_PROTOCOL_FREE_33               = 33, /* FREE */
-  NDPI_PROTOCOL_FREE_34               = 34, /* FREE */
+  NDPI_PROTOCOL_YANDEX_MAIL           = 33,
+  NDPI_PROTOCOL_YANDEX_MUSIC          = 34,
   NDPI_PROTOCOL_GNUTELLA              = 35,
   NDPI_PROTOCOL_EDONKEY               = 36,
   NDPI_PROTOCOL_BITTORRENT            = 37,
@@ -84,13 +84,13 @@ typedef enum {
   NDPI_PROTOCOL_CPHA                  = 53,
   NDPI_PROTOCOL_PPSTREAM              = 54,
   NDPI_PROTOCOL_ZATTOO                = 55,
-  NDPI_PROTOCOL_FREE_56               = 56, /* FREE */
-  NDPI_PROTOCOL_FREE_57               = 57, /* FREE */
+  NDPI_PROTOCOL_YANDEX_MARKET         = 56,
+  NDPI_PROTOCOL_YANDEX_DISK           = 57,
   NDPI_PROTOCOL_DISCORD               = 58,
   NDPI_PROTOCOL_TVUPLAYER             = 59,
   NDPI_PROTOCOL_MONGODB               = 60,
   NDPI_PROTOCOL_PLURALSIGHT           = 61,
-  NDPI_PROTOCOL_FREE_62               = 62, /* FREE */
+  NDPI_PROTOCOL_YANDEX_CLOUD          = 62,
   NDPI_PROTOCOL_OCSP                  = 63,
   NDPI_PROTOCOL_VXLAN                 = 64,
   NDPI_PROTOCOL_IRC                   = 65,
@@ -126,8 +126,8 @@ typedef enum {
   NDPI_PROTOCOL_IAX                   = 95,
   NDPI_PROTOCOL_TFTP                  = 96,
   NDPI_PROTOCOL_AFP                   = 97,
-  NDPI_PROTOCOL_FREE_98               = 98, /* FREE */
-  NDPI_PROTOCOL_FREE_99               = 99, /* FREE */
+  NDPI_PROTOCOL_YANDEX_METRIKA        = 98,
+  NDPI_PROTOCOL_YANDEX_DIRECT         = 99,
   NDPI_PROTOCOL_SIP                   = 100,
   NDPI_PROTOCOL_TRUPHONE              = 101,
   NDPI_PROTOCOL_IP_ICMPV6             = 102,
diff --git a/src/lib/inc_generated/ndpi_asn_yandex.c.inc b/src/lib/inc_generated/ndpi_asn_yandex.c.inc
new file mode 100644
index 000000000..64c8b76fb
--- /dev/null
+++ b/src/lib/inc_generated/ndpi_asn_yandex.c.inc
@@ -0,0 +1,45 @@
+/*
+ *
+ * This file is generated automatically and part of nDPI
+ *
+ * nDPI is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * nDPI is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with nDPI.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+/* ****************************************************** */
+
+
+static ndpi_network ndpi_protocol_yandex_protocol_list[] = {
+ { 0x052DC000 /* 5.45.192.0/18 */, 18, NDPI_PROTOCOL_YANDEX },
+ { 0x05FFC000 /* 5.255.192.0/18 */, 18, NDPI_PROTOCOL_YANDEX },
+ { 0x25094000 /* 37.9.64.0/18 */, 18, NDPI_PROTOCOL_YANDEX },
+ { 0x258C8000 /* 37.140.128.0/18 */, 18, NDPI_PROTOCOL_YANDEX },
+ { 0x4D580000 /* 77.88.0.0/18 */, 18, NDPI_PROTOCOL_YANDEX },
+ { 0x54FCA000 /* 84.252.160.0/19 */, 19, NDPI_PROTOCOL_YANDEX },
+ { 0x57FAE000 /* 87.250.224.0/19 */, 19, NDPI_PROTOCOL_YANDEX },
+ { 0x5A9CB000 /* 90.156.176.0/22 */, 22, NDPI_PROTOCOL_YANDEX },
+ { 0x5A9CB400 /* 90.156.180.0/23 */, 23, NDPI_PROTOCOL_YANDEX },
+ { 0x5A9CB600 /* 90.156.182.0/24 */, 24, NDPI_PROTOCOL_YANDEX },
+ { 0x5D9E8000 /* 93.158.128.0/18 */, 18, NDPI_PROTOCOL_YANDEX },
+ { 0x5F6C8000 /* 95.108.128.0/17 */, 17, NDPI_PROTOCOL_YANDEX },
+ { 0x642B4000 /* 100.43.64.0/19 */, 19, NDPI_PROTOCOL_YANDEX },
+ { 0x8D088000 /* 141.8.128.0/18 */, 18, NDPI_PROTOCOL_YANDEX },
+ { 0xB29A8000 /* 178.154.128.0/18 */, 18, NDPI_PROTOCOL_YANDEX },
+ { 0xB920B800 /* 185.32.184.0/22 */, 22, NDPI_PROTOCOL_YANDEX },
+ { 0xC7156000 /* 199.21.96.0/22 */, 22, NDPI_PROTOCOL_YANDEX },
+ { 0xC724F000 /* 199.36.240.0/22 */, 22, NDPI_PROTOCOL_YANDEX },
+ { 0xD5B4C000 /* 213.180.192.0/19 */, 19, NDPI_PROTOCOL_YANDEX },
+ /* End */
+ { 0x0, 0, 0 }
+};
diff --git a/src/lib/inc_generated/ndpi_asn_yandex_cloud.c.inc b/src/lib/inc_generated/ndpi_asn_yandex_cloud.c.inc
new file mode 100644
index 000000000..2fbfce512
--- /dev/null
+++ b/src/lib/inc_generated/ndpi_asn_yandex_cloud.c.inc
@@ -0,0 +1,37 @@
+/*
+ *
+ * This file is generated automatically and part of nDPI
+ *
+ * nDPI is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * nDPI is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with nDPI.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+/* ****************************************************** */
+
+
+static ndpi_network ndpi_protocol_yandex_cloud_protocol_list[] = {
+ { 0x33FA0000 /* 51.250.0.0/17 */, 17, NDPI_PROTOCOL_YANDEX_CLOUD },
+ { 0x3E547000 /* 62.84.112.0/20 */, 20, NDPI_PROTOCOL_YANDEX_CLOUD },
+ { 0x54C98000 /* 84.201.128.0/18 */, 18, NDPI_PROTOCOL_YANDEX_CLOUD },
+ { 0x54FC8000 /* 84.252.128.0/20 */, 20, NDPI_PROTOCOL_YANDEX_CLOUD },
+ { 0x59A98000 /* 89.169.128.0/18 */, 18, NDPI_PROTOCOL_YANDEX_CLOUD },
+ { 0x82C12000 /* 130.193.32.0/19 */, 19, NDPI_PROTOCOL_YANDEX_CLOUD },
+ { 0x9EA00000 /* 158.160.0.0/16 */, 16, NDPI_PROTOCOL_YANDEX_CLOUD },
+ { 0xB29AC000 /* 178.154.192.0/18 */, 18, NDPI_PROTOCOL_YANDEX_CLOUD },
+ { 0xB9CEA400 /* 185.206.164.0/22 */, 22, NDPI_PROTOCOL_YANDEX_CLOUD },
+ { 0xC120D800 /* 193.32.216.0/22 */, 22, NDPI_PROTOCOL_YANDEX_CLOUD },
+ { 0xD91CE000 /* 217.28.224.0/20 */, 20, NDPI_PROTOCOL_YANDEX_CLOUD },
+ /* End */
+ { 0x0, 0, 0 }
+};
diff --git a/src/lib/ndpi_content_match.c.inc b/src/lib/ndpi_content_match.c.inc
index 618b7b162..e971c4e0b 100644
--- a/src/lib/ndpi_content_match.c.inc
+++ b/src/lib/ndpi_content_match.c.inc
@@ -1798,6 +1798,16 @@ static ndpi_protocol_match host_match[] =
    { "vkuseraudio.net",                  "VK", NDPI_PROTOCOL_VK, NDPI_PROTOCOL_CATEGORY_SOCIAL_NETWORK, NDPI_PROTOCOL_FUN, NDPI_PROTOCOL_DEFAULT_LEVEL },
    { "vkuservideo.net",                  "VK", NDPI_PROTOCOL_VK, NDPI_PROTOCOL_CATEGORY_SOCIAL_NETWORK, NDPI_PROTOCOL_FUN, NDPI_PROTOCOL_DEFAULT_LEVEL },
 
+   { "yandex.",                          "Yandex", NDPI_PROTOCOL_YANDEX, NDPI_PROTOCOL_CATEGORY_WEB, NDPI_PROTOCOL_SAFE, NDPI_PROTOCOL_DEFAULT_LEVEL },
+   { "yastatic.net",                     "Yandex", NDPI_PROTOCOL_YANDEX, NDPI_PROTOCOL_CATEGORY_WEB, NDPI_PROTOCOL_SAFE, NDPI_PROTOCOL_DEFAULT_LEVEL },
+   { "mail.yandex.",                     "YandexMail", NDPI_PROTOCOL_YANDEX_MAIL, NDPI_PROTOCOL_CATEGORY_MAIL, NDPI_PROTOCOL_SAFE, NDPI_PROTOCOL_DEFAULT_LEVEL },
+   { "music.yandex.",                    "YandexMusic", NDPI_PROTOCOL_YANDEX_MUSIC, NDPI_PROTOCOL_CATEGORY_MUSIC, NDPI_PROTOCOL_FUN, NDPI_PROTOCOL_DEFAULT_LEVEL },
+   { "market.yandex.",                   "YandexMarket", NDPI_PROTOCOL_YANDEX_MARKET, NDPI_PROTOCOL_CATEGORY_SHOPPING, NDPI_PROTOCOL_SAFE, NDPI_PROTOCOL_DEFAULT_LEVEL },
+   { "disk.yandex.",                     "YandexDisk", NDPI_PROTOCOL_YANDEX_DISK, NDPI_PROTOCOL_CATEGORY_CLOUD, NDPI_PROTOCOL_SAFE, NDPI_PROTOCOL_DEFAULT_LEVEL },
+   { "cloud.yandex.",                    "YandexCloud", NDPI_PROTOCOL_YANDEX_CLOUD, NDPI_PROTOCOL_CATEGORY_CLOUD, NDPI_PROTOCOL_SAFE, NDPI_PROTOCOL_DEFAULT_LEVEL },
+   { "metrika.yandex.",                  "YandexMetrika", NDPI_PROTOCOL_YANDEX_METRIKA, NDPI_PROTOCOL_CATEGORY_WEB, NDPI_PROTOCOL_SAFE, NDPI_PROTOCOL_DEFAULT_LEVEL },
+   { "direct.yandex.",                   "YandexDirect", NDPI_PROTOCOL_YANDEX_DIRECT, CUSTOM_CATEGORY_ADVERTISEMENT, NDPI_PROTOCOL_TRACKER_ADS, NDPI_PROTOCOL_DEFAULT_LEVEL },
+
 #ifdef CUSTOM_NDPI_PROTOCOLS
 #include "../../../nDPI-custom/custom_ndpi_content_match_host_match.c.inc"
 #endif
@@ -1820,6 +1830,8 @@ static ndpi_tls_cert_name_match tls_certificate_match [] = {
   { "CN=www.update.microsoft.com",     NDPI_PROTOCOL_WINDOWS_UPDATE },
   { "CN=*.tunnelbear.com",             NDPI_PROTOCOL_TUNNELBEAR },
   { "CN=cloudflareclient.com",         NDPI_PROTOCOL_CLOUDFLARE_WARP },
+  { "O=V Kontakte LLC",                NDPI_PROTOCOL_VK },
+  { "O=Yandex LLC",                    NDPI_PROTOCOL_YANDEX },
 
   { NULL, 0 }
 };
diff --git a/src/lib/ndpi_main.c b/src/lib/ndpi_main.c
index 6d8da0b78..d3037bb76 100644
--- a/src/lib/ndpi_main.c
+++ b/src/lib/ndpi_main.c
@@ -101,7 +101,8 @@
 #include "inc_generated/ndpi_asn_discord.c.inc"
 #include "inc_generated/ndpi_asn_line.c.inc"
 #include "inc_generated/ndpi_asn_vk.c.inc"
-
+#include "inc_generated/ndpi_asn_yandex.c.inc"
+#include "inc_generated/ndpi_asn_yandex_cloud.c.inc"
 
 /* Third party libraries */
 #include "third_party/include/ndpi_patricia.h"
@@ -1154,10 +1155,6 @@ static void ndpi_init_protocol_defaults(struct ndpi_detection_module_struct *ndp
 			  "MySQL", NDPI_PROTOCOL_CATEGORY_DATABASE,
 			  ndpi_build_default_ports(ports_a, 3306, 0, 0, 0, 0) /* TCP */,
 			  ndpi_build_default_ports(ports_b, 0, 0, 0, 0, 0) /* UDP */);
-  ndpi_set_proto_defaults(ndpi_str, 1 /* cleartext */, 0 /* nw proto */, NDPI_PROTOCOL_POTENTIALLY_DANGEROUS, NDPI_PROTOCOL_FREE_25,
-			  "Free25", NDPI_PROTOCOL_CATEGORY_DOWNLOAD_FT,
-			  ndpi_build_default_ports(ports_a, 0, 0, 0, 0, 0) /* TCP */,
-			  ndpi_build_default_ports(ports_b, 0, 0, 0, 0, 0) /* UDP */);
   ndpi_set_proto_defaults(ndpi_str, 0 /* encrypted */, 0 /* nw proto */, NDPI_PROTOCOL_ACCEPTABLE, NDPI_PROTOCOL_NATS,
 			  "Nats", NDPI_PROTOCOL_CATEGORY_RPC,
 			  ndpi_build_default_ports(ports_a, 0, 0, 0, 0, 0) /* TCP */,
@@ -1178,14 +1175,6 @@ static void ndpi_init_protocol_defaults(struct ndpi_detection_module_struct *ndp
 			  "Kontiki", NDPI_PROTOCOL_CATEGORY_MEDIA,
 			  ndpi_build_default_ports(ports_a, 0, 0, 0, 0, 0) /* TCP */,
 			  ndpi_build_default_ports(ports_b, 0, 0, 0, 0, 0) /* UDP */);
-  ndpi_set_proto_defaults(ndpi_str, 1 /* cleartext */, 0 /* nw proto */, NDPI_PROTOCOL_POTENTIALLY_DANGEROUS, NDPI_PROTOCOL_FREE_33,
-			  "Free33", NDPI_PROTOCOL_CATEGORY_DOWNLOAD_FT,
-			  ndpi_build_default_ports(ports_a, 0, 0, 0, 0, 0) /* TCP */,
-			  ndpi_build_default_ports(ports_b, 0, 0, 0, 0, 0) /* UDP */);
-  ndpi_set_proto_defaults(ndpi_str, 1 /* cleartext */, 0 /* nw proto */, NDPI_PROTOCOL_POTENTIALLY_DANGEROUS, NDPI_PROTOCOL_FREE_34,
-			  "Free34", NDPI_PROTOCOL_CATEGORY_DOWNLOAD_FT,
-			  ndpi_build_default_ports(ports_a, 0, 0, 0, 0, 0) /* TCP */,
-			  ndpi_build_default_ports(ports_b, 0, 0, 0, 0, 0) /* UDP */);
   ndpi_set_proto_defaults(ndpi_str, 1 /* cleartext */, 0 /* nw proto */, NDPI_PROTOCOL_POTENTIALLY_DANGEROUS, NDPI_PROTOCOL_GNUTELLA,
 			  "Gnutella", NDPI_PROTOCOL_CATEGORY_DOWNLOAD_FT,
 			  ndpi_build_default_ports(ports_a, 0, 0, 0, 0, 0) /* TCP */,
@@ -1298,14 +1287,6 @@ static void ndpi_init_protocol_defaults(struct ndpi_detection_module_struct *ndp
 			  "Zattoo", NDPI_PROTOCOL_CATEGORY_VIDEO,
 			  ndpi_build_default_ports(ports_a, 0, 0, 0, 0, 0) /* TCP */,
 			  ndpi_build_default_ports(ports_b, 0, 0, 0, 0, 0) /* UDP */);
-  ndpi_set_proto_defaults(ndpi_str, 1 /* cleartext */, 0 /* nw proto */, NDPI_PROTOCOL_FUN, NDPI_PROTOCOL_FREE_56,
-			  "Free56", NDPI_PROTOCOL_CATEGORY_MUSIC,
-			  ndpi_build_default_ports(ports_a, 0, 0, 0, 0, 0) /* TCP */,
-			  ndpi_build_default_ports(ports_b, 0, 0, 0, 0, 0) /* UDP */);
-  ndpi_set_proto_defaults(ndpi_str, 1 /* cleartext */, 0 /* nw proto */, NDPI_PROTOCOL_FUN, NDPI_PROTOCOL_FREE_57,
-			  "Free57", NDPI_PROTOCOL_CATEGORY_VIDEO,
-			  ndpi_build_default_ports(ports_a, 0, 0, 0, 0, 0) /* TCP */,
-			  ndpi_build_default_ports(ports_b, 0, 0, 0, 0, 0) /* UDP */);
   ndpi_set_proto_defaults(ndpi_str, 0 /* encrypted */, 1 /* app proto */, NDPI_PROTOCOL_FUN, NDPI_PROTOCOL_DISCORD,
 			  "Discord", NDPI_PROTOCOL_CATEGORY_COLLABORATIVE,
 			  ndpi_build_default_ports(ports_a, 0, 0, 0, 0, 0) /* TCP */,
@@ -1318,10 +1299,6 @@ static void ndpi_init_protocol_defaults(struct ndpi_detection_module_struct *ndp
 			  "Pluralsight", NDPI_PROTOCOL_CATEGORY_VIDEO,
 			  ndpi_build_default_ports(ports_a, 0, 0, 0, 0, 0) /* TCP */,
 			  ndpi_build_default_ports(ports_b, 0, 0, 0, 0, 0) /* UDP */);
-  ndpi_set_proto_defaults(ndpi_str, 1 /* cleartext */, 1 /* app proto */, NDPI_PROTOCOL_FUN, NDPI_PROTOCOL_FREE_62,
-			  "Free62", NDPI_PROTOCOL_CATEGORY_DOWNLOAD_FT,
-			  ndpi_build_default_ports(ports_a, 0, 0, 0, 0, 0) /* TCP */,
-			  ndpi_build_default_ports(ports_b, 0, 0, 0, 0, 0) /* UDP */);
   ndpi_set_proto_defaults(ndpi_str, 1 /* cleartext */, 1 /* app proto */, NDPI_PROTOCOL_SAFE, NDPI_PROTOCOL_OCSP,
 			  "OCSP", NDPI_PROTOCOL_CATEGORY_NETWORK,
 			  ndpi_build_default_ports(ports_a, 0, 0, 0, 0, 0) /* TCP */,
@@ -1480,14 +1457,6 @@ static void ndpi_init_protocol_defaults(struct ndpi_detection_module_struct *ndp
 			  "CHECKMK", NDPI_PROTOCOL_CATEGORY_DATA_TRANSFER,
 			  ndpi_build_default_ports(ports_a, 6556, 0, 0, 0, 0) /* TCP */,
 			  ndpi_build_default_ports(ports_b, 0, 0, 0, 0, 0) /* UDP */);
-  ndpi_set_proto_defaults(ndpi_str, 1 /* cleartext */, 0 /* nw proto */, NDPI_PROTOCOL_POTENTIALLY_DANGEROUS, NDPI_PROTOCOL_FREE_98,
-			  "Free98", NDPI_PROTOCOL_CATEGORY_DOWNLOAD_FT,
-			  ndpi_build_default_ports(ports_a, 0, 0, 0, 0, 0) /* TCP */,
-			  ndpi_build_default_ports(ports_b, 0, 0, 0, 0, 0) /* UDP */);
-  ndpi_set_proto_defaults(ndpi_str, 1 /* cleartext */, 1 /* app proto */, NDPI_PROTOCOL_FUN, NDPI_PROTOCOL_FREE_99,
-			  "Free99", NDPI_PROTOCOL_CATEGORY_DOWNLOAD_FT,
-			  ndpi_build_default_ports(ports_a, 0, 0, 0, 0, 0) /* TCP */,
-			  ndpi_build_default_ports(ports_b, 0, 0, 0, 0, 0) /* UDP */);
   ndpi_set_proto_defaults(ndpi_str, 1 /* cleartext */, 0 /* nw proto */, NDPI_PROTOCOL_ACCEPTABLE, NDPI_PROTOCOL_SIP,
 			  "SIP", NDPI_PROTOCOL_CATEGORY_VOIP,
 			  ndpi_build_default_ports(ports_a, 5060, 5061, 0, 0, 0) /* TCP */,
@@ -2788,6 +2757,8 @@ struct ndpi_detection_module_struct *ndpi_init_detection_module(ndpi_init_prefs
       ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_discord_protocol_list);
       ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_line_protocol_list);
       ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_vk_protocol_list);
+      ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_yandex_protocol_list);
+      ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_yandex_cloud_protocol_list);
     }
 
     if(prefs & ndpi_track_flow_payload)