Implemented FacebookZero protocol support

https://code.facebook.com/posts/608854979307125/building-zero-protocol-for-fast-secure-mobile-connections/

Discontinued filetopia protocol

7 files changed, 140 insertions, 108 deletions

diff --git a/src/include/ndpi_protocol_ids.h b/src/include/ndpi_protocol_ids.h
index d1f29ea7f..c626d3d9f 100644
--- a/src/include/ndpi_protocol_ids.h
+++ b/src/include/ndpi_protocol_ids.h
@@ -67,7 +67,7 @@
 #define NDPI_PROTOCOL_COAP                  27
 #define NDPI_PROTOCOL_VMWARE                28
 #define NDPI_PROTOCOL_MAIL_SMTPS            29
-#define NDPI_PROTOCOL_FILETOPIA             30
+#define NDPI_PROTOCOL_FBZERO                30
 #define NDPI_PROTOCOL_UBNTAC2               31 /* Ubiquity UBNT AirControl 2 - Thomas Fjellstrom <thomas+ndpi@fjellstrom.ca> */
 #define NDPI_PROTOCOL_KONTIKI               32
 #define NDPI_PROTOCOL_OPENFT                33
diff --git a/src/include/ndpi_protocols.h b/src/include/ndpi_protocols.h
index 189b35a8e..f08b1de73 100644
--- a/src/include/ndpi_protocols.h
+++ b/src/include/ndpi_protocols.h
@@ -232,7 +232,6 @@ void init_eaq_dissector(struct ndpi_detection_module_struct *ndpi_struct, u_int3
 void init_edonkey_dissector(struct ndpi_detection_module_struct *ndpi_struct, u_int32_t *id, NDPI_PROTOCOL_BITMASK *detection_bitmask);
 void init_fasttrack_dissector(struct ndpi_detection_module_struct *ndpi_struct, u_int32_t *id, NDPI_PROTOCOL_BITMASK *detection_bitmask);
 void init_fiesta_dissector(struct ndpi_detection_module_struct *ndpi_struct, u_int32_t *id, NDPI_PROTOCOL_BITMASK *detection_bitmask);
-void init_filetopia_dissector(struct ndpi_detection_module_struct *ndpi_struct, u_int32_t *id, NDPI_PROTOCOL_BITMASK *detection_bitmask);
 void init_florensia_dissector(struct ndpi_detection_module_struct *ndpi_struct, u_int32_t *id, NDPI_PROTOCOL_BITMASK *detection_bitmask);
 void init_ftp_control_dissector(struct ndpi_detection_module_struct *ndpi_struct, u_int32_t *id, NDPI_PROTOCOL_BITMASK *detection_bitmask);
 void init_ftp_data_dissector(struct ndpi_detection_module_struct *ndpi_struct, u_int32_t *id, NDPI_PROTOCOL_BITMASK *detection_bitmask);
@@ -356,4 +355,5 @@ void init_checkmk_dissector(struct ndpi_detection_module_struct *ndpi_struct, u_
 void init_apple_push_dissector(struct ndpi_detection_module_struct *ndpi_struct, u_int32_t *id, NDPI_PROTOCOL_BITMASK *detection_bitmask);
 void init_whatsapp_dissector(struct ndpi_detection_module_struct *ndpi_struct, u_int32_t *id, NDPI_PROTOCOL_BITMASK *detection_bitmask);
 void init_ajp_dissector(struct ndpi_detection_module_struct *ndpi_struct, u_int32_t *id, NDPI_PROTOCOL_BITMASK *detection_bitmask);
+void init_fbzero_dissector(struct ndpi_detection_module_struct *ndpi_struct, u_int32_t *id, NDPI_PROTOCOL_BITMASK *detection_bitmask);
 #endif /* __NDPI_PROTOCOLS_H__ */
diff --git a/src/include/ndpi_typedefs.h b/src/include/ndpi_typedefs.h
index 2bffb4001..9ec6b7158 100644
--- a/src/include/ndpi_typedefs.h
+++ b/src/include/ndpi_typedefs.h
@@ -490,6 +490,7 @@ struct ndpi_flow_tcp_struct {
   u_int8_t irc_stage;
   u_int8_t irc_port;
 #endif
+  
 #ifdef NDPI_PROTOCOL_H323
   u_int8_t h323_valid_packets;
 #endif
@@ -505,9 +506,6 @@ struct ndpi_flow_tcp_struct {
 #ifdef NDPI_PROTOCOL_SOULSEEK
   u_int32_t soulseek_stage:2;
 #endif
-#ifdef NDPI_PROTOCOL_FILETOPIA
-  u_int32_t filetopia_stage:2;
-#endif
 #ifdef NDPI_PROTOCOL_TDS
   u_int32_t tds_stage:3;
 #endif
diff --git a/src/lib/Makefile.am b/src/lib/Makefile.am
index b1ff5a072..633c110d8 100644
--- a/src/lib/Makefile.am
+++ b/src/lib/Makefile.am
@@ -51,9 +51,9 @@ libndpi_la_SOURCES = ndpi_content_match.c.inc \
 		     protocols/eaq.c \
 		     protocols/edonkey.c \
 		     protocols/fasttrack.c \
+		     protocols/fbzero.c \
 		     protocols/fiesta.c \
 		     protocols/fix.c \
-		     protocols/filetopia.c \
 		     protocols/florensia.c \
 		     protocols/ftp_control.c \
 		     protocols/ftp_data.c \
diff --git a/src/lib/ndpi_main.c b/src/lib/ndpi_main.c
index 28976fe56..7767f0b74 100644
--- a/src/lib/ndpi_main.c
+++ b/src/lib/ndpi_main.c
@@ -1082,10 +1082,10 @@ static void ndpi_init_protocol_defaults(struct ndpi_detection_module_struct *ndp
 			    no_master, "VMware", NDPI_PROTOCOL_CATEGORY_REMOTE_ACCESS,
 			    ndpi_build_default_ports(ports_a, 903, 0, 0, 0, 0) /* TCP */,
 			    ndpi_build_default_ports(ports_b, 902, 903, 0, 0, 0) /* UDP */);
-    ndpi_set_proto_defaults(ndpi_mod, NDPI_PROTOCOL_POTENTIALLY_DANGEROUS, NDPI_PROTOCOL_FILETOPIA,
+    ndpi_set_proto_defaults(ndpi_mod, NDPI_PROTOCOL_SAFE, NDPI_PROTOCOL_FBZERO,
 			    no_master,
-			    no_master, "Filetopia", NDPI_PROTOCOL_CATEGORY_DOWNLOAD_FT,
-			    ndpi_build_default_ports(ports_a, 0, 0, 0, 0, 0) /* TCP */,
+			    no_master, "FacebookZero", NDPI_PROTOCOL_CATEGORY_WEB,
+			    ndpi_build_default_ports(ports_a, 443, 0, 0, 0, 0) /* TCP */,
 			    ndpi_build_default_ports(ports_b, 0, 0, 0, 0, 0) /* UDP */);
     ndpi_set_proto_defaults(ndpi_mod, NDPI_PROTOCOL_POTENTIALLY_DANGEROUS, NDPI_PROTOCOL_KONTIKI,
 			    no_master,
@@ -2754,7 +2754,7 @@ void ndpi_set_protocol_detection_bitmask2(struct ndpi_detection_module_struct *n
   init_dns_dissector(ndpi_struct, &a, detection_bitmask);
 
   /* FILETOPIA */
-  init_filetopia_dissector(ndpi_struct, &a, detection_bitmask);
+  init_fbzero_dissector(ndpi_struct, &a, detection_bitmask);
 
   /* VMWARE */
   init_vmware_dissector(ndpi_struct, &a, detection_bitmask);
diff --git a/src/lib/protocols/fbzero.c b/src/lib/protocols/fbzero.c
new file mode 100644
index 000000000..31acd0c9a
--- /dev/null
+++ b/src/lib/protocols/fbzero.c
@@ -0,0 +1,132 @@
+/*
+ * fbzero.c
+ *
+ * Copyright (C) 2018 - ntop.org
+ *
+ * nDPI is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * nDPI is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with nDPI.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "ndpi_protocol_ids.h"
+
+/* https://code.facebook.com/posts/608854979307125/building-zero-protocol-for-fast-secure-mobile-connections/ */
+
+#ifdef NDPI_PROTOCOL_FBZERO
+
+#define NDPI_CURRENT_PROTO NDPI_PROTOCOL_FBZERO
+
+#include "ndpi_api.h"
+
+PACK_ON
+struct fbzero_tag {
+  char tag[4];
+  u_int32_t tag_offset_len;
+} PACK_OFF;
+
+PACK_ON
+struct fbzero_header {
+  u_int8_t flags;
+  char version[3];
+  u_int8_t unknown;
+  u_int32_t len;
+  char tag[4];
+  u_int16_t tag_number, _pad;
+} PACK_OFF;
+
+/* **************************************************************************** */
+
+void ndpi_search_fbzero(struct ndpi_detection_module_struct *ndpi_struct,
+			struct ndpi_flow_struct *flow) {
+  struct ndpi_packet_struct *packet = &flow->packet;
+
+  NDPI_LOG_DBG(ndpi_struct, "search FacebookZero\n");
+
+  if(packet->payload_packet_len > sizeof(struct fbzero_header)) {
+    struct fbzero_header *h = (struct fbzero_header*)packet->payload;
+    struct fbzero_tag *t;
+    u_int offset = sizeof(struct fbzero_header), i, data_offset, tag_number, data_prev_offset = 0;
+
+    if((h->flags & 0x01) == 0)
+      goto fbzero_not_found;
+
+    if((h->version[0] != 'Q')
+       || (h->version[1] != 'T')
+       || (h->version[2] != 'V'))
+      goto fbzero_not_found;
+
+    if(h->unknown != 0x30)
+      goto fbzero_not_found;
+
+    t = (struct fbzero_tag*)&packet->payload[offset];
+    tag_number = h->tag_number;
+    data_offset = offset + tag_number*sizeof(struct fbzero_tag);
+
+    if(strncmp(h->tag, "CHLO", 4))
+      goto fbzero_not_found;
+
+    for(i=0; i<h->tag_number; i++) {
+#ifdef DEBUG
+      printf("[FBZERO] %u) %c%c%c%c\n", i, t->tag[0], t->tag[1], t->tag[2], t->tag[3]);
+#endif
+
+      offset += sizeof(struct fbzero_tag);
+
+      if((t->tag[0] == 'S') && (t->tag[1] == 'N')
+	 && (t->tag[2] == 'I') && (t->tag[3] == '\0')) {
+	char *value = (char*)&packet->payload[data_offset + data_prev_offset];
+	u_int tag_len = t->tag_offset_len-data_prev_offset, max_len;
+
+	max_len = ndpi_min(tag_len, sizeof(flow->host_server_name)-1);
+
+	strncpy((char*)flow->host_server_name, value, max_len);
+	flow->host_server_name[max_len] = '\0';
+
+#ifdef DEBUG
+	printf("[FBZERO] SNI [@%u][len: %u][%s]\n", tag_len,
+	       t->tag_offset_len-data_prev_offset, flow->host_server_name);
+#endif
+
+	ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_FBZERO, NDPI_PROTOCOL_UNKNOWN);
+	  
+	ndpi_match_host_subprotocol(ndpi_struct, flow, (char *)flow->host_server_name,
+				    strlen((const char *)flow->host_server_name),
+				    NDPI_PROTOCOL_FBZERO);
+	return;
+      }
+
+      data_prev_offset = t->tag_offset_len;
+      t = (struct fbzero_tag*)&packet->payload[offset];
+    }
+
+    return;
+  }
+
+ fbzero_not_found:
+  NDPI_EXCLUDE_PROTO(ndpi_struct, flow);
+}
+
+/* **************************************************************************** */
+
+void init_fbzero_dissector(struct ndpi_detection_module_struct *ndpi_struct, u_int32_t *id,
+			   NDPI_PROTOCOL_BITMASK *detection_bitmask) {
+  ndpi_set_bitmask_protocol_detection("FacebookZero", ndpi_struct, detection_bitmask, *id,
+				      NDPI_PROTOCOL_FBZERO,
+				      ndpi_search_fbzero,
+				      NDPI_SELECTION_BITMASK_PROTOCOL_V4_V6_TCP_WITH_PAYLOAD_WITHOUT_RETRANSMISSION,
+				      SAVE_DETECTION_BITMASK_AS_UNKNOWN,
+				      ADD_TO_DETECTION_BITMASK);
+  *id += 1;
+}
+
+#endif
diff --git a/src/lib/protocols/filetopia.c b/src/lib/protocols/filetopia.c
deleted file mode 100644
index eb3215e48..000000000
--- a/src/lib/protocols/filetopia.c
+++ /dev/null
@@ -1,98 +0,0 @@
-/*
- * filetopia.c
- *
- * Copyright (C) 2009-2011 by ipoque GmbH
- * Copyright (C) 2011-15 - ntop.org
- *
- * This file is part of nDPI, an open source deep packet inspection
- * library based on the OpenDPI and PACE technology by ipoque GmbH
- *
- * nDPI is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Lesser General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * nDPI is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public License
- * along with nDPI.  If not, see <http://www.gnu.org/licenses/>.
- *
- */
-
-#include "ndpi_protocol_ids.h"
-
-#ifdef NDPI_PROTOCOL_FILETOPIA
-
-#define NDPI_CURRENT_PROTO NDPI_PROTOCOL_FILETOPIA
-
-#include "ndpi_api.h"
-
-
-static void ndpi_int_filetopia_add_connection(struct ndpi_detection_module_struct *ndpi_struct, struct ndpi_flow_struct *flow)
-{
-  ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_FILETOPIA, NDPI_PROTOCOL_UNKNOWN);
-}
-
-void ndpi_search_filetopia_tcp(struct ndpi_detection_module_struct *ndpi_struct, struct ndpi_flow_struct *flow)
-{
-	struct ndpi_packet_struct *packet = &flow->packet;
-	
-	NDPI_LOG_DBG(ndpi_struct, "search Filetopia\n");
-
-	if (flow->l4.tcp.filetopia_stage == 0) {
-		if (packet->payload_packet_len >= 50 && packet->payload_packet_len <= 70
-			&& packet->payload[0] == 0x03 && packet->payload[1] == 0x9a
-			&& packet->payload[3] == 0x22 && packet->payload[packet->payload_packet_len - 1] == 0x2b) {
-			NDPI_LOG_DBG2(ndpi_struct, "Filetopia stage 1 detected\n");
-			flow->l4.tcp.filetopia_stage = 1;
-			return;
-		}
-
-	} else if (flow->l4.tcp.filetopia_stage == 1) {
-		if (packet->payload_packet_len >= 100 && packet->payload[0] == 0x03
-			&& packet->payload[1] == 0x9a && (packet->payload[3] == 0x22 || packet->payload[3] == 0x23)) {
-
-			int i;
-			for (i = 0; i < 10; i++) {	// check 10 bytes for valid ASCII printable characters
-				if (!(packet->payload[5 + i] >= 0x20 && packet->payload[5 + i] <= 0x7e)) {
-					goto end_filetopia_nothing_found;
-				}
-			}
-
-			NDPI_LOG_DBG2(ndpi_struct, "Filetopia stage 2 detected\n");
-			flow->l4.tcp.filetopia_stage = 2;
-			return;
-		}
-
-
-	} else if (flow->l4.tcp.filetopia_stage == 2) {
-		if (packet->payload_packet_len >= 4 && packet->payload_packet_len <= 100
-			&& packet->payload[0] == 0x03 && packet->payload[1] == 0x9a
-			&& (packet->payload[3] == 0x22 || packet->payload[3] == 0x23)) {
-			NDPI_LOG_INFO(ndpi_struct, "found Filetopia\n");
-			ndpi_int_filetopia_add_connection(ndpi_struct, flow);
-			return;
-		}
-
-	}
-
-  end_filetopia_nothing_found:
-	NDPI_EXCLUDE_PROTO(ndpi_struct, flow);
-}
-
-
-void init_filetopia_dissector(struct ndpi_detection_module_struct *ndpi_struct, u_int32_t *id, NDPI_PROTOCOL_BITMASK *detection_bitmask)
-{
-  ndpi_set_bitmask_protocol_detection("Filetopia", ndpi_struct, detection_bitmask, *id,
-				      NDPI_PROTOCOL_FILETOPIA,
-				      ndpi_search_filetopia_tcp,
-				      NDPI_SELECTION_BITMASK_PROTOCOL_V4_V6_TCP_WITH_PAYLOAD_WITHOUT_RETRANSMISSION,
-				      SAVE_DETECTION_BITMASK_AS_UNKNOWN,
-				      ADD_TO_DETECTION_BITMASK);
-  *id += 1;
-}
-
-#endif