Merge remote-tracking branch 'upstream' into dev

9 files changed, 140 insertions, 29 deletions

diff --git a/example/ndpiReader.c b/example/ndpiReader.c
index 8551992bf..6351ce034 100644
--- a/example/ndpiReader.c
+++ b/example/ndpiReader.c
@@ -843,7 +843,7 @@ static void printFlow(u_int16_t id, struct ndpi_flow_info *flow, u_int16_t threa
     if(flow->ssh_ssl.ja3_server[0] != '\0') fprintf(out, "[JA3S: %s%s]", flow->ssh_ssl.ja3_server,
 						    print_cipher(flow->ssh_ssl.server_unsafe_cipher));
     if(flow->ssh_ssl.server_organization[0] != '\0') fprintf(out, "[organization: %s]", flow->ssh_ssl.server_organization);
-    if(flow->ssh_ssl.server_cipher != '\0') fprintf(out, "[%s]", ndpi_cipher2str(flow->ssh_ssl.server_cipher));;
+    if(flow->ssh_ssl.server_cipher != '\0') fprintf(out, "[Cipher: %s]", ndpi_cipher2str(flow->ssh_ssl.server_cipher));
 
     if(flow->bittorent_hash[0] != '\0') fprintf(out, "[BT Hash: %s]", flow->bittorent_hash);
     if(flow->dhcp_fingerprint[0] != '\0') fprintf(out, "[DHCP Fingerprint: %s]", flow->dhcp_fingerprint);
@@ -2474,6 +2474,8 @@ static void ndpi_process_packet(u_char *args,
   /* allocate an exact size buffer to check overflows */
   uint8_t *packet_checked = malloc(header->caplen);
 
+  /* ndpi_has_human_readeable_string(ndpi_info_mod, (char*)packet, header->caplen, 3); */
+  
   memcpy(packet_checked, packet, header->caplen);
   p = ndpi_workflow_process_packet(ndpi_thread_info[thread_id].workflow, header, packet_checked);
 
diff --git a/src/include/ndpi_api.h b/src/include/ndpi_api.h
index 631ab4038..7297f0038 100644
--- a/src/include/ndpi_api.h
+++ b/src/include/ndpi_api.h
@@ -811,7 +811,8 @@ extern "C" {
   const char* ndpi_cipher2str(u_int32_t cipher);
   u_int16_t ndpi_guess_host_protocol_id(struct ndpi_detection_module_struct *ndpi_struct,
 					struct ndpi_flow_struct *flow);
-
+  int ndpi_has_human_readeable_string(struct ndpi_detection_module_struct *ndpi_struct,
+				      char *buffer, u_int buffer_size);
 #ifdef __cplusplus
 }
 #endif
diff --git a/src/include/ndpi_protocol_ids.h b/src/include/ndpi_protocol_ids.h
index b8b77ada4..918ae3886 100644
--- a/src/include/ndpi_protocol_ids.h
+++ b/src/include/ndpi_protocol_ids.h
@@ -80,9 +80,7 @@ typedef enum {
   NDPI_PROTOCOL_NEST_LOG_SINK         = 43, /* Nest Log Sink (Nest Protect) - Darryl Sokoloski <darryl@egloo.ca> */
   NDPI_PROTOCOL_MODBUS                = 44, /* Modbus */
   NDPI_PROTOCOL_WHATSAPP_VIDEO        = 45,
-
-  NDPI_PROTOCOL_FREE_46               = 46, /* Free */
-  
+  NDPI_PROTOCOL_DATASAVER             = 46, /* Protocols used to save data on Internet communications */  
   NDPI_PROTOCOL_XBOX                  = 47,
   NDPI_PROTOCOL_QQ                    = 48,
   NDPI_PROTOCOL_TIKTOK                = 49,
@@ -232,7 +230,7 @@ typedef enum {
   NDPI_PROTOCOL_KAKAOTALK             = 193, /* KakaoTalk Chat (no voice call) */
   NDPI_PROTOCOL_KAKAOTALK_VOICE       = 194, /* KakaoTalk Voice */
   NDPI_PROTOCOL_TWITCH                = 195, /* Edoardo Dominici <edoaramis@gmail.com> */
-  NDPI_PROTOCOL_FREE_196              = 196, /* Free */
+  NDPI_PROTOCOL_DNS_OVER_HTTPS        = 196,
   NDPI_PROTOCOL_WECHAT                = 197,
   NDPI_PROTOCOL_MPEGTS                = 198,
   NDPI_PROTOCOL_SNAPCHAT              = 199,
diff --git a/src/lib/Makefile.in b/src/lib/Makefile.in
index aa2bbcc31..1eb7abc59 100644
--- a/src/lib/Makefile.in
+++ b/src/lib/Makefile.in
@@ -46,8 +46,8 @@ $(NDPI_LIB_STATIC): $(OBJECTS)
 
 $(NDPI_LIB_SHARED): $(OBJECTS)
 	$(CC) -shared -fPIC $(SONAME_FLAG) -o $@ $(OBJECTS)
-	ln -Ffs $(NDPI_LIB_SHARED) $(NDPI_LIB_SHARED_BASE)
-	ln -Ffs $(NDPI_LIB_SHARED) $(NDPI_LIB_SHARED_BASE).$(NDPI_VERSION_MAJOR)
+	ln -fs $(NDPI_LIB_SHARED) $(NDPI_LIB_SHARED_BASE)
+	ln -fs $(NDPI_LIB_SHARED) $(NDPI_LIB_SHARED_BASE).$(NDPI_VERSION_MAJOR)
 
 %.o: %.c $(HEADERS) Makefile
 	$(CC) $(CFLAGS) -c $< -o $@
diff --git a/src/lib/ndpi_content_match.c.inc b/src/lib/ndpi_content_match.c.inc
index a58d80e25..208853c59 100644
--- a/src/lib/ndpi_content_match.c.inc
+++ b/src/lib/ndpi_content_match.c.inc
@@ -8367,6 +8367,24 @@ ndpi_protocol_match host_match[] = {
 
   { "android.clients.google.com", NULL, "android\\.clients\\.google" TLD,   "PlayStore",    NDPI_PROTOCOL_PLAYSTORE, NDPI_PROTOCOL_CATEGORY_SW_UPDATE, NDPI_PROTOCOL_SAFE },
 
+    /*
+    https://www.fastvue.co/sophos/blog/google-data-saver-affect-security-confidentiality-reporting/
+    Used by Google Chrome Lite Mode for Android
+
+    This traffic will bypass checks and blocks as it will include all the communications from/to
+    the browser instead of using the standard communication mechanisms SSL/HTTP(S)/DNS
+  */
+  { ".googlezip.net", NULL, ".googlezip\\.net" TLD,                       "DataSaver",     NDPI_PROTOCOL_DATASAVER, NDPI_PROTOCOL_CATEGORY_WEB, NDPI_PROTOCOL_ACCEPTABLE },	 
+  { "datasaver.googleapis.com", NULL, "datasaver\\.googleapis\\.com" TLD, "DataSaver",     NDPI_PROTOCOL_DATASAVER, NDPI_PROTOCOL_CATEGORY_WEB, NDPI_PROTOCOL_ACCEPTABLE },
+
+  /* http://check.googlezip.net/connect [check browser connectivity] */
+  // { ".googlezip.net", NULL, "\\.googlezip" TLD,                   "Google",           NDPI_PROTOCOL_GOOGLE, NDPI_PROTOCOL_CATEGORY_WEB, NDPI_PROTOCOL_SAFE },
+
+  { "dns.google", NULL, "dns\\.google" TLD, "DNSoverHTTPS",     NDPI_PROTOCOL_DNS_OVER_HTTPS, NDPI_PROTOCOL_CATEGORY_NETWORK, NDPI_PROTOCOL_ACCEPTABLE },
+  // { "mozilla.cloudflare-dns.com", NULL, "mozilla\\.cloudflare-dns\\.com" TLD, "DNSoverHTTPS",     NDPI_PROTOCOL_DNS_OVER_HTTPS, NDPI_PROTOCOL_CATEGORY_NETWORK, NDPI_PROTOCOL_ACCEPTABLE }, /* Firefox */
+  { "cloudflare-dns.com", NULL, "cloudflare-dns\\.com" TLD, "DNSoverHTTPS",     NDPI_PROTOCOL_DNS_OVER_HTTPS, NDPI_PROTOCOL_CATEGORY_NETWORK, NDPI_PROTOCOL_ACCEPTABLE },
+
+
   /*
     See https://better.fyi/trackers/
 
@@ -8407,9 +8425,6 @@ ndpi_protocol_match host_match[] = {
   { "plus.google.com", NULL, "plus\\.google" TLD,                 "GooglePlus",       NDPI_PROTOCOL_GOOGLE_PLUS, NDPI_PROTOCOL_CATEGORY_SOCIAL_NETWORK, NDPI_PROTOCOL_FUN },
   { "plus.url.google.com", NULL, "plus\\.url\\.google" TLD,       "GooglePlus",       NDPI_PROTOCOL_GOOGLE_PLUS, NDPI_PROTOCOL_CATEGORY_SOCIAL_NETWORK, NDPI_PROTOCOL_FUN },
 
-  /* http://check.googlezip.net/connect [check browser connectivity] */
-  { ".googlezip.net", NULL, "\\.googlezip" TLD,                   "Google",           NDPI_PROTOCOL_GOOGLE, NDPI_PROTOCOL_CATEGORY_WEB, NDPI_PROTOCOL_SAFE },
-
   { "googleusercontent.com", NULL, "googleusercontent" TLD,       "Google",           NDPI_PROTOCOL_GOOGLE, NDPI_PROTOCOL_CATEGORY_WEB, NDPI_PROTOCOL_ACCEPTABLE },
   { "1e100.net", NULL, "1e100" TLD,                               "Google",           NDPI_PROTOCOL_GOOGLE, NDPI_PROTOCOL_CATEGORY_WEB, NDPI_PROTOCOL_ACCEPTABLE },
 
@@ -8687,11 +8702,10 @@ ndpi_protocol_match host_match[] = {
   { "whispersystems.org", NULL, "whispersystems\\.org" TLD,            "Signal",           NDPI_PROTOCOL_SIGNAL, NDPI_PROTOCOL_CATEGORY_CHAT, NDPI_PROTOCOL_ACCEPTABLE },
   { "musical.ly", NULL, "musical\\.ly" TLD,                            "TikTok",           NDPI_PROTOCOL_TIKTOK, NDPI_PROTOCOL_CATEGORY_SOCIAL_NETWORK, NDPI_PROTOCOL_ACCEPTABLE },
   { "muscdn.com", NULL, "muscndl\\.com" TLD,                           "TikTok",           NDPI_PROTOCOL_TIKTOK, NDPI_PROTOCOL_CATEGORY_SOCIAL_NETWORK, NDPI_PROTOCOL_ACCEPTABLE },
-  
+
   { NULL, NULL, NULL, 0 }
 };
-
-/* ****************************************************** */
+/* ******************************************************************** */
 
 /*
   Tor
@@ -8753,6 +8767,8 @@ static const char *ndpi_en_bigrams[] = {
   "py", "ry", "sy", "ty", "uy", "vy", "wy", "xy", "yy", "zy", "az", "bz", "cz", "dz", "ez", "gz", "iz",
   "lz", "nz", "oz", "pz", "rz", "tz", "uz", "zz", NULL };
 
+/* ******************************************************************** */
+
 static const char *ndpi_en_impossible_bigrams[] = {
   "bk", "bq", "bx", "cb", "cf", "cg", "cj", "cp", "cv", "cw", "cx", "dx", "fk", "fq", "fv", "fx", /* "ee", removed it can be found in 'meeting' */
   "fz", "gq", "gv", "gx", "hh", "hk", "hv", "hx", "hz", "iy", "jb", /* "jc", jcrew.com */ "jd", "jf", "jg", "jh", "jk",
@@ -8765,3 +8781,5 @@ static const char *ndpi_en_impossible_bigrams[] = {
   "yj", "yq", "yv", "yz", "yw", "zb", "zc", "zg", "zh", "zj", "zn", "zq", "zs", "zx", "wh", "wk",
   "wb", "zk", "kp", "zk", "xy",
   NULL };
+
+
diff --git a/src/lib/ndpi_main.c b/src/lib/ndpi_main.c
index 3a420607e..3cf0f3d1e 100644
--- a/src/lib/ndpi_main.c
+++ b/src/lib/ndpi_main.c
@@ -650,8 +650,7 @@ static void destroy_hyperscan(struct ndpi_detection_module_struct *ndpi_mod) {
 
 /* ******************************************************************** */
 
-static void init_string_based_protocols(struct ndpi_detection_module_struct *ndpi_mod)
-{
+static void init_string_based_protocols(struct ndpi_detection_module_struct *ndpi_mod) {
   int i;
 
 #ifdef HAVE_HYPERSCAN
@@ -988,9 +987,9 @@ static void ndpi_init_protocol_defaults(struct ndpi_detection_module_struct *ndp
 			    no_master, "WhatsAppVideo", NDPI_PROTOCOL_CATEGORY_VOIP,
 			    ndpi_build_default_ports(ports_a, 0, 0, 0, 0, 0) /* TCP */,
 			    ndpi_build_default_ports(ports_b, 0, 0, 0, 0, 0) /* UDP */);
-    ndpi_set_proto_defaults(ndpi_mod, NDPI_PROTOCOL_FUN, NDPI_PROTOCOL_FREE_46,
+    ndpi_set_proto_defaults(ndpi_mod, NDPI_PROTOCOL_FUN, NDPI_PROTOCOL_DATASAVER,
 			    0 /* can_have_a_subprotocol */, no_master,
-			    no_master, "Free", NDPI_PROTOCOL_CATEGORY_CUSTOM_1 /* dummy */,
+			    no_master, "DataSaver", NDPI_PROTOCOL_CATEGORY_WEB /* dummy */,
 			    ndpi_build_default_ports(ports_a, 0, 0, 0, 0, 0) /* TCP */,
 			    ndpi_build_default_ports(ports_b, 0, 0, 0, 0, 0) /* UDP */);
     ndpi_set_proto_defaults(ndpi_mod, NDPI_PROTOCOL_FUN, NDPI_PROTOCOL_SIGNAL,
@@ -998,9 +997,9 @@ static void ndpi_init_protocol_defaults(struct ndpi_detection_module_struct *ndp
 			    no_master, "Signal", NDPI_PROTOCOL_CATEGORY_CHAT,
 			    ndpi_build_default_ports(ports_a, 0, 0, 0, 0, 0) /* TCP */,
 			    ndpi_build_default_ports(ports_b, 0, 0, 0, 0, 0) /* UDP */);
-    ndpi_set_proto_defaults(ndpi_mod, NDPI_PROTOCOL_FUN, NDPI_PROTOCOL_FREE_196,
+    ndpi_set_proto_defaults(ndpi_mod, NDPI_PROTOCOL_FUN, NDPI_PROTOCOL_DNS_OVER_HTTPS,
 			    0 /* can_have_a_subprotocol */, no_master,
-			    no_master, "Free", NDPI_PROTOCOL_CATEGORY_CUSTOM_1 /* dummy */,
+			    no_master, "DNSoverHTTPS", NDPI_PROTOCOL_CATEGORY_NETWORK /* dummy */,
 			    ndpi_build_default_ports(ports_a, 0, 0, 0, 0, 0) /* TCP */,
 			    ndpi_build_default_ports(ports_b, 0, 0, 0, 0, 0) /* UDP */);
     ndpi_set_proto_defaults(ndpi_mod, NDPI_PROTOCOL_FUN, NDPI_PROTOCOL_FREE_205,
@@ -3613,7 +3612,7 @@ void ndpi_connection_tracking(struct ndpi_detection_module_struct *ndpi_struct,
   if(ndpi_struct->direction_detect_disable) {
     packet->packet_direction = flow->packet_direction;
   } else {
-    if(iph != NULL && le32toh(iph->saddr) < le32toh(iph->daddr))
+    if(iph != NULL && ntohl(iph->saddr) < ntohl(iph->daddr))
       packet->packet_direction = 1;
 
 #ifdef NDPI_DETECTION_SUPPORT_IPV6
@@ -3635,7 +3634,7 @@ void ndpi_connection_tracking(struct ndpi_detection_module_struct *ndpi_struct,
     packet->num_retried_bytes = 0;
 
     if(!ndpi_struct->direction_detect_disable)
-      packet->packet_direction = (le16toh(tcph->source) < le16toh(tcph->dest)) ? 1 : 0;
+      packet->packet_direction = (ntohs(tcph->source) < ntohs(tcph->dest)) ? 1 : 0;
 
     if(tcph->syn != 0 && tcph->ack == 0 && flow->l4.tcp.seen_syn == 0 && flow->l4.tcp.seen_syn_ack == 0
        && flow->l4.tcp.seen_ack == 0) {
@@ -3698,7 +3697,7 @@ void ndpi_connection_tracking(struct ndpi_detection_module_struct *ndpi_struct,
     }
   } else if(udph != NULL) {
     if(!ndpi_struct->direction_detect_disable)
-      packet->packet_direction = (le16toh(udph->source) < le16toh(udph->dest)) ? 1 : 0;
+      packet->packet_direction = (htons(udph->source) < htons(udph->dest)) ? 1 : 0;
   }
 
   if(flow->packet_counter < MAX_PACKET_COUNTER && packet->payload_packet_len) {
@@ -4588,6 +4587,7 @@ ndpi_protocol ndpi_detection_process_packet(struct ndpi_detection_module_struct
      && (ret.app_protocol == NDPI_PROTOCOL_UNKNOWN)
      && flow->packet.tcp
      && (flow->packet.tcp->syn == 0)
+     && (flow->guessed_protocol_id == 0)
      ) {
     /*
       This is a TCP flow
diff --git a/src/lib/ndpi_utils.c b/src/lib/ndpi_utils.c
index bcd8e5f48..480f4ff18 100644
--- a/src/lib/ndpi_utils.c
+++ b/src/lib/ndpi_utils.c
@@ -66,7 +66,7 @@ int ndpi_check_punycode_string(char * buffer , int len) {
       // is a punycode string
       return(1);
   }
-  
+
   // not a punycode string
   return 0;
 }
@@ -276,7 +276,7 @@ int strcasecmp(const char *s1, const char *s2) {
   register const u_char *cm = charmap,
     *us1 = (const u_char *)s1,
     *us2 = (const u_char *)s2;
-  
+
   while (cm[*us1] == cm[*us2++])
     if(*us1++ == '\0')
       return (0);
@@ -590,7 +590,95 @@ const char* ndpi_cipher2str(u_int32_t cipher) {
   case 0x060040: return("SSL2_DES_64_CBC_WITH_MD5");
   case 0x0700c0: return("SSL2_DES_192_EDE3_CBC_WITH_MD5");
   case 0x080080: return("SSL2_RC4_64_WITH_MD5");
-  default: return("???");
+  case 0x001301: return("TLS_AES_128_GMC_SHA256");
+  default:
+    {
+      static char buf[8];
+
+      snprintf(buf, sizeof(buf), "0X%04X", cipher);
+      return(buf);
+    }
   }
 }
 
+/* ******************************************************************** */
+
+static int ndpi_is_other_char(char c) {
+  return((c == '.')
+	 || (c == '@')
+	 );
+}
+
+/* ******************************************************************** */
+
+static int ndpi_is_valid_char(char c) {
+  return(isdigit(c)
+	 || isalpha(c)
+	 || ndpi_is_other_char(c));
+}
+
+/* ******************************************************************** */
+
+
+static int ndpi_find_non_eng_bigrams(struct ndpi_detection_module_struct *ndpi_struct,
+				     char *str) {
+  char s[3];
+
+  if((isdigit(str[0]) && isdigit(str[1]))
+     || (ndpi_is_other_char(str[0]) || ndpi_is_other_char(str[1]))
+     || (ndpi_is_other_char(str[0]) || ndpi_is_other_char(str[1]))
+     )
+    return(1);
+
+  s[0] = tolower(str[0]), s[1] = tolower(str[1]), s[2] = '\0';
+
+  return(ndpi_match_bigram(ndpi_struct, &ndpi_struct->bigrams_automa, s));
+}
+
+/* ******************************************************************** */
+
+/* #define PRINT_STRINGS 1 */
+
+int ndpi_has_human_readeable_string(struct ndpi_detection_module_struct *ndpi_struct,
+				    char *buffer, u_int buffer_size) {
+  u_int ret = 0, i = 0, do_cr = 0, len = 0;
+  const u_int8_t NDPI_MIN_VALID_STRING_LEN = 4; /* Will return 0 if no string > NDPI_MIN_VALID_STRING_LEN have been found */
+
+  if(buffer_size <= 0)
+    return(0);
+
+  for(i=0; i<buffer_size-2; i++) {
+    if(ndpi_is_valid_char(buffer[i])
+       && ndpi_is_valid_char(buffer[i+1])
+       && ndpi_find_non_eng_bigrams(ndpi_struct, &buffer[i])) {
+#ifdef PRINT_STRINGS
+      printf("%c%c", buffer[i], buffer[i+1]);
+#endif
+      do_cr = 1, i += 1, len += 2;
+    } else {
+      if(ndpi_is_valid_char(buffer[i]) && do_cr) {
+#ifdef PRINT_STRINGS
+	printf("%c", buffer[i]);
+#endif
+	len += 1;
+      }
+
+      // printf("->> %c%c\n", isprint(buffer[i]) ? buffer[i] : '.', isprint(buffer[i+1]) ? buffer[i+1] : '.');
+      if(do_cr) {
+#ifdef PRINT_STRINGS
+	printf(" [len: %u]\n", len);
+#endif
+	if(len > NDPI_MIN_VALID_STRING_LEN)
+	  ret = 1;
+
+	do_cr = 0, len = 0;
+      }
+    }
+  }
+
+#ifdef PRINT_STRINGS
+  printf("=======>> Found string: %u\n", ret);
+#endif
+
+  return(ret);
+}
diff --git a/src/lib/protocols/quic.c b/src/lib/protocols/quic.c
index d14538e0d..d67414fdf 100644
--- a/src/lib/protocols/quic.c
+++ b/src/lib/protocols/quic.c
@@ -100,8 +100,8 @@ void ndpi_search_quic(struct ndpi_detection_module_struct *ndpi_struct,
 	     && (packet->payload[i+1] == 'N')
 	     && (packet->payload[i+2] == 'I')
 	     && (packet->payload[i+3] == 0)) {
-	    u_int32_t offset = le32toh(*((u_int32_t*)&packet->payload[i+4]));
-	    u_int32_t prev_offset = le32toh(*((u_int32_t*)&packet->payload[i-4]));
+	    u_int32_t offset = ntohl(*((u_int32_t*)&packet->payload[i+4]));
+	    u_int32_t prev_offset = ntohl(*((u_int32_t*)&packet->payload[i-4]));
 	    int len = offset-prev_offset;
 	    int sni_offset = i+prev_offset+1;
 
diff --git a/src/lib/protocols/ssl.c b/src/lib/protocols/ssl.c
index 947b8371d..f7009a788 100644
--- a/src/lib/protocols/ssl.c
+++ b/src/lib/protocols/ssl.c
@@ -339,6 +339,10 @@ static void stripCertificateTrailer(char *buffer, int buffer_len) {
 	buffer[i] = '\0', buffer_len = i;
     }
   }
+
+  /* Now all lowecase */
+  for(i=0; i<buffer_len; i++)
+    buffer[i] = tolower(buffer[i]);
 }
 
 /* https://engineering.salesforce.com/tls-fingerprinting-with-ja3-and-ja3s-247362855967 */
@@ -498,7 +502,7 @@ int getSSLcertificate(struct ndpi_detection_module_struct *ndpi_struct,
 	      if(num_found != 2) continue;
 	    }
 
-	    if(server_len+i+3 < packet->payload_packet_len) {
+	    if((server_len+i+3) < packet->payload_packet_len) {
 	      char *server_name = (char*)&packet->payload[i+4];
 	      u_int8_t begin = 0, len, j, num_dots;