RTMP: improve detection (#2549)

7 files changed, 54 insertions, 39 deletions

diff --git a/doc/protocols.rst b/doc/protocols.rst
index df8952c90..6a6116385 100644
--- a/doc/protocols.rst
+++ b/doc/protocols.rst
@@ -82,6 +82,15 @@ Notes:
 - IPv6 is not supported
 
 
+.. _Proto 174:
+
+`NDPI_PROTOCOL_RTMP`
+====================
+Real-Time Messaging Protocol (RTMP) is a communication protocol for streaming audio, video, and data over the Internet.
+
+References: `Wikipedia <https://en.wikipedia.org/wiki/Real-Time_Messaging_Protocol>`_
+
+
 .. _Proto 182:
 
 `NDPI_PROTOCOL_RESP`
diff --git a/src/include/ndpi_typedefs.h b/src/include/ndpi_typedefs.h
index b8a8ddf4f..e292633fd 100644
--- a/src/include/ndpi_typedefs.h
+++ b/src/include/ndpi_typedefs.h
@@ -841,6 +841,10 @@ struct ndpi_flow_tcp_struct {
   /* NDPI_PROTOCOL_TELNET */
   u_int32_t telnet_stage:2;			// 0 - 2
 
+  /* NDPI_PROTOCOL_RTMP */
+  u_int32_t rtmp_stage:2;
+  u_int16_t rtmp_client_buffer_len;
+
   struct {
     /* NDPI_PROTOCOL_TLS */
     u_int8_t app_data_seen[2];
@@ -1493,9 +1497,6 @@ struct ndpi_flow_struct {
   /* NDPI_PROTOCOL_FTP_CONTROL */
   u_int8_t ftp_control_stage:2;
 
-  /* NDPI_PROTOCOL_RTMP */
-  u_int8_t rtmp_stage:2;
-
   /* NDPI_PROTOCOL_STARCRAFT */
   u_int8_t starcraft_udp_stage : 3;	// 0-7
 
diff --git a/src/lib/protocols/rtmp.c b/src/lib/protocols/rtmp.c
index c3799248d..a1da9760c 100644
--- a/src/lib/protocols/rtmp.c
+++ b/src/lib/protocols/rtmp.c
@@ -39,45 +39,48 @@ static void ndpi_int_rtmp_add_connection(struct ndpi_detection_module_struct *nd
 static void ndpi_check_rtmp(struct ndpi_detection_module_struct *ndpi_struct, struct ndpi_flow_struct *flow)
 {
   struct ndpi_packet_struct *packet = &ndpi_struct->packet;
-  u_int32_t payload_len = packet->payload_packet_len;
   
-  /* Break after 13 packets. */
-  if (flow->packet_counter > 13) {
+  /* Look for the handshake, which is only at the beginning of the flow:
+      C->S: 0x03 + 1536 bytes
+      S->C: 0X03 + something...; we don't really check the length of the burst sent by the server, to avoid to save further state
+     See: https://en.wikipedia.org/w/index.php?title=Real-Time_Messaging_Protocol&section=12#Handshake */
+
+  if(!ndpi_seen_flow_beginning(flow)) {
     NDPI_EXCLUDE_PROTO(ndpi_struct, flow);
     return;
   }
-  
-  /* Check if we so far detected the protocol in the request or not. */
-  if(flow->rtmp_stage == 0) {
+
+  /* TODO: should we check somehow for mid-flows? */
+
+  if(flow->l4.tcp.rtmp_stage == 0) {
     NDPI_LOG_DBG2(ndpi_struct, "RTMP stage 0: \n");
      
-    if ((payload_len >= 9) &&
-	((packet->payload[0] == 0x03) || (packet->payload[0] == 0x06)) &&
-	/* https://en.wikipedia.org/w/index.php?title=Real-Time_Messaging_Protocol&section=12#Handshake */
-	get_u_int32_t(packet->payload, 5) == 0) {
-      NDPI_LOG_DBG2(ndpi_struct, "Possible RTMP request detected, we will look further for the response\n");
-       
-      /* Encode the direction of the packet in the stage, so we will know when we need to look for the response packet. */
-      flow->rtmp_stage = packet->packet_direction + 1;
-    } else
-      NDPI_EXCLUDE_PROTO(ndpi_struct, flow);
+    if(packet->payload[0] == 0x03) {
+      flow->l4.tcp.rtmp_stage = packet->packet_direction + 1;
+      flow->l4.tcp.rtmp_client_buffer_len = packet->payload_packet_len;
+      return;
+    }
   } else {
-    NDPI_LOG_DBG2(ndpi_struct, "RTMP stage %u: \n", flow->rtmp_stage);
-    
+    NDPI_LOG_DBG2(ndpi_struct, "RTMP stage %u (client already sent %d bytes)\n",
+                  flow->l4.tcp.rtmp_stage, flow->l4.tcp.rtmp_client_buffer_len);
+
     /* At first check, if this is for sure a response packet (in another direction. If not, do nothing now and return. */
-    if ((flow->rtmp_stage - packet->packet_direction) == 1) {
-      return;
+    if(flow->l4.tcp.rtmp_stage - packet->packet_direction == 1) {
+      /* From the same direction */
+      flow->l4.tcp.rtmp_client_buffer_len += packet->payload_packet_len;
+      if(flow->l4.tcp.rtmp_client_buffer_len <= 1537)
+        return;
     }
-    
-    /* This is a packet in another direction. Check if we find the proper response. */
-    if ((payload_len >= 4) && ((packet->payload[0] == 0x03) || (packet->payload[0] == 0x06) || (packet->payload[0] == 0x08) || (packet->payload[0] == 0x09) || (packet->payload[0] == 0x0a))) {
+
+    /* This is a packet in another direction */
+    if(packet->payload[0] == 0x03 && flow->l4.tcp.rtmp_client_buffer_len == 1537) {
       NDPI_LOG_INFO(ndpi_struct, "found RTMP\n");
       ndpi_int_rtmp_add_connection(ndpi_struct, flow);
-    } else {
-      NDPI_LOG_DBG2(ndpi_struct, "The reply did not seem to belong to RTMP, resetting the stage to 0\n");
-      flow->rtmp_stage = 0;
+      return;
     }
   }
+
+  NDPI_EXCLUDE_PROTO(ndpi_struct, flow);
 }
 
 static void ndpi_search_rtmp(struct ndpi_detection_module_struct *ndpi_struct, struct ndpi_flow_struct *flow)
diff --git a/tests/cfgs/default/pcap/rtmp.pcap b/tests/cfgs/default/pcap/rtmp.pcap
index 4aa67353b..12f7b9880 100644
--- a/tests/cfgs/default/pcap/rtmp.pcap
+++ b/tests/cfgs/default/pcap/rtmp.pcap
diff --git a/tests/cfgs/default/result/iso9506-1-mms.pcap.out b/tests/cfgs/default/result/iso9506-1-mms.pcap.out
index c020cb15e..2adf3c673 100644
--- a/tests/cfgs/default/result/iso9506-1-mms.pcap.out
+++ b/tests/cfgs/default/result/iso9506-1-mms.pcap.out
@@ -1,6 +1,6 @@
 DPI Packets (TCP):	7	(7.00 pkts/flow)
 Confidence DPI              : 1 (flows)
-Num dissector calls: 196 (196.00 diss/flow)
+Num dissector calls: 197 (197.00 diss/flow)
 LRU cache ookla:      0/0/0 (insert/search/found)
 LRU cache bittorrent: 0/0/0 (insert/search/found)
 LRU cache stun:       0/0/0 (insert/search/found)
diff --git a/tests/cfgs/default/result/rtmp.pcap.out b/tests/cfgs/default/result/rtmp.pcap.out
index 0e2bc7464..6e59d7be7 100644
--- a/tests/cfgs/default/result/rtmp.pcap.out
+++ b/tests/cfgs/default/result/rtmp.pcap.out
@@ -1,13 +1,13 @@
-DPI Packets (TCP):	8	(8.00 pkts/flow)
-Confidence DPI              : 1 (flows)
-Num dissector calls: 180 (180.00 diss/flow)
+DPI Packets (TCP):	18	(6.00 pkts/flow)
+Confidence DPI              : 3 (flows)
+Num dissector calls: 551 (183.67 diss/flow)
 LRU cache ookla:      0/0/0 (insert/search/found)
 LRU cache bittorrent: 0/0/0 (insert/search/found)
 LRU cache stun:       0/0/0 (insert/search/found)
 LRU cache tls_cert:   0/0/0 (insert/search/found)
 LRU cache mining:     0/0/0 (insert/search/found)
 LRU cache msteams:    0/0/0 (insert/search/found)
-LRU cache fpc_dns:    0/1/0 (insert/search/found)
+LRU cache fpc_dns:    0/3/0 (insert/search/found)
 Automa host:          0/0 (search/found)
 Automa domain:        0/0 (search/found)
 Automa tls cert:      0/0 (search/found)
@@ -17,11 +17,13 @@ Patricia risk mask:   0/0 (search/found)
 Patricia risk mask IPv6: 0/0 (search/found)
 Patricia risk:        0/0 (search/found)
 Patricia risk IPv6:   0/0 (search/found)
-Patricia protocols:   2/0 (search/found)
+Patricia protocols:   5/1 (search/found)
 Patricia protocols IPv6: 0/0 (search/found)
 
-RTMP	26	8368	1
+RTMP	60	25463	3
 
-Acceptable                      26 8368          1            
+Acceptable                      60 25463         3            
 
-	1	TCP 192.168.43.1:1177 <-> 192.168.43.128:1935 [proto: 174/RTMP][IP: 0/Unknown][ClearText][Confidence: DPI][FPC: 0/Unknown, Confidence: Unknown][DPI packets: 8][cat: Media/1][12 pkts/4108 bytes <-> 14 pkts/4260 bytes][Goodput ratio: 84/82][1.04 sec][bytes ratio: -0.018 (Mixed)][IAT c2s/s2c min/avg/max/stddev: 0/0 83/75 334/230 119/85][Pkt Len c2s/s2c min/avg/max/stddev: 54/54 342/304 1514/1514 531/465][PLAIN TEXT (ace@nAt)][Plen Bins: 7,21,14,0,7,7,0,0,0,7,0,7,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,7,0,0,0,0,0,21,0,0]
+	1	TCP 10.140.74.206:59509 <-> 193.118.47.46:6002 [VLAN: 113][proto: GTP:174/RTMP][IP: 0/Unknown][ClearText][Confidence: DPI][FPC: 0/Unknown, Confidence: Unknown][DPI packets: 5][cat: Media/1][8 pkts/4268 bytes <-> 9 pkts/4410 bytes][Goodput ratio: 79/77][2.62 sec][bytes ratio: -0.016 (Mixed)][IAT c2s/s2c min/avg/max/stddev: 0/0 331/225 819/1319 326/455][Pkt Len c2s/s2c min/avg/max/stddev: 122/118 534/490 1458/1458 539/530][Risk: ** Known Proto on Non Std Port **][Risk Score: 50][PLAIN TEXT (StreamMedia)][Plen Bins: 13,20,6,0,0,13,6,6,0,0,0,6,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,27,0,0,0,0,0]
+	2	TCP 10.140.80.204:49757 <-> 203.205.224.48:1935 [VLAN: 113][proto: GTP:174/RTMP][IP: 285/Tencent][ClearText][Confidence: DPI][FPC: 285/Tencent, Confidence: IP address][DPI packets: 5][cat: Media/1][8 pkts/4155 bytes <-> 9 pkts/4262 bytes][Goodput ratio: 81/79][0.83 sec][bytes ratio: -0.013 (Mixed)][IAT c2s/s2c min/avg/max/stddev: 0/0 98/72 177/168 76/54][Pkt Len c2s/s2c min/avg/max/stddev: 110/99 519/474 1458/1458 547/539][PLAIN TEXT (connect)][Plen Bins: 26,13,0,0,0,13,0,13,0,0,0,6,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,26,0,0,0,0,0]
+	3	TCP 192.168.43.1:1177 <-> 192.168.43.128:1935 [proto: 174/RTMP][IP: 0/Unknown][ClearText][Confidence: DPI][FPC: 0/Unknown, Confidence: Unknown][DPI packets: 8][cat: Media/1][12 pkts/4108 bytes <-> 14 pkts/4260 bytes][Goodput ratio: 84/82][1.04 sec][bytes ratio: -0.018 (Mixed)][IAT c2s/s2c min/avg/max/stddev: 0/0 83/75 334/230 119/85][Pkt Len c2s/s2c min/avg/max/stddev: 54/54 342/304 1514/1514 531/465][PLAIN TEXT (ace@nAt)][Plen Bins: 7,21,14,0,7,7,0,0,0,7,0,7,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,7,0,0,0,0,0,21,0,0]
diff --git a/tests/cfgs/default/result/s7comm-plus.pcap.out b/tests/cfgs/default/result/s7comm-plus.pcap.out
index 3a0e58c8d..0d91c27be 100644
--- a/tests/cfgs/default/result/s7comm-plus.pcap.out
+++ b/tests/cfgs/default/result/s7comm-plus.pcap.out
@@ -1,6 +1,6 @@
 DPI Packets (TCP):	9	(9.00 pkts/flow)
 Confidence DPI              : 1 (flows)
-Num dissector calls: 191 (191.00 diss/flow)
+Num dissector calls: 192 (192.00 diss/flow)
 LRU cache ookla:      0/0/0 (insert/search/found)
 LRU cache bittorrent: 0/0/0 (insert/search/found)
 LRU cache stun:       0/0/0 (insert/search/found)