Added Z39.50 protocol.add/z3950

Signed-off-by: Toni Uhlig <matzeton@googlemail.com>

8 files changed, 170 insertions, 2 deletions

diff --git a/src/include/ndpi_protocol_ids.h b/src/include/ndpi_protocol_ids.h
index 66cfef54c..ed567c080 100644
--- a/src/include/ndpi_protocol_ids.h
+++ b/src/include/ndpi_protocol_ids.h
@@ -288,6 +288,7 @@ typedef enum {
   NDPI_PROTOCOL_GENSHIN_IMPACT        = 257, /* Toni Uhlig <matzeton@googlemail.com> */
   NDPI_PROTOCOL_ACTIVISION            = 258,
   NDPI_PROTOCOL_FORTICLIENT           = 259,
+  NDPI_PROTOCOL_Z3950                 = 260, /* Toni Uhlig <matzeton@googlemail.com> */
 
 #ifdef CUSTOM_NDPI_PROTOCOLS
 #include "../../../nDPI-custom/custom_ndpi_protocol_ids.h"
diff --git a/src/include/ndpi_protocols.h b/src/include/ndpi_protocols.h
index 959a6d643..ff507d717 100644
--- a/src/include/ndpi_protocols.h
+++ b/src/include/ndpi_protocols.h
@@ -214,5 +214,6 @@ void init_mongodb_dissector(struct ndpi_detection_module_struct *ndpi_struct, u_
 void init_among_us_dissector(struct ndpi_detection_module_struct *ndpi_struct, u_int32_t *id, NDPI_PROTOCOL_BITMASK *detection_bitmask);
 void init_hpvirtgrp_dissector(struct ndpi_detection_module_struct *ndpi_struct, u_int32_t *id, NDPI_PROTOCOL_BITMASK *detection_bitmask);
 void init_genshin_impact_dissector(struct ndpi_detection_module_struct *ndpi_struct, u_int32_t *id, NDPI_PROTOCOL_BITMASK *detection_bitmask);
+void init_z3950_dissector(struct ndpi_detection_module_struct *ndpi_struct, u_int32_t *id, NDPI_PROTOCOL_BITMASK *detection_bitmask);
 
 #endif /* __NDPI_PROTOCOLS_H__ */
diff --git a/src/include/ndpi_typedefs.h b/src/include/ndpi_typedefs.h
index 173f3baff..ffe12e24c 100644
--- a/src/include/ndpi_typedefs.h
+++ b/src/include/ndpi_typedefs.h
@@ -1473,6 +1473,9 @@ struct ndpi_flow_struct {
   /* NDPI_PROTOCOL_STARCRAFT */
   u_int8_t starcraft_udp_stage : 3;	// 0-7
 
+  /* NDPI_PROTOCOL_Z3950 */
+  u_int8_t z3950_stage : 2; // 0-3
+
   /* NDPI_PROTOCOL_OPENVPN */
   u_int8_t ovpn_session_id[8];
   u_int8_t ovpn_counter;
diff --git a/src/lib/ndpi_main.c b/src/lib/ndpi_main.c
index ff4b0cb5d..648fdbfee 100644
--- a/src/lib/ndpi_main.c
+++ b/src/lib/ndpi_main.c
@@ -1681,6 +1681,11 @@ static void ndpi_init_protocol_defaults(struct ndpi_detection_module_struct *ndp
 			  "FortiClient", NDPI_PROTOCOL_CATEGORY_VPN,
 			  ndpi_build_default_ports(ports_a, 0, 0, 0, 0, 0) /* TCP */,
 			  ndpi_build_default_ports(ports_b, 0, 0, 0, 0, 0) /* UDP */);
+  ndpi_set_proto_defaults(ndpi_str, NDPI_PROTOCOL_ACCEPTABLE, NDPI_PROTOCOL_Z3950,
+              "Z39.50",
+              NDPI_PROTOCOL_CATEGORY_NETWORK,
+              ndpi_build_default_ports(ports_a, 210, 0, 0, 0, 0) /* TCP */,
+              ndpi_build_default_ports(ports_b, 0, 0, 0, 0, 0) /* UDP */);
 
 #ifdef CUSTOM_NDPI_PROTOCOLS
 #include "../../../nDPI-custom/custom_ndpi_main.c"
@@ -3871,6 +3876,9 @@ void ndpi_set_protocol_detection_bitmask2(struct ndpi_detection_module_struct *n
   /* Genshin Impact */
   init_genshin_impact_dissector(ndpi_str, &a, detection_bitmask);
 
+  /* Z39.50 international standard client–server, application layer communications protocol */
+  init_z3950_dissector(ndpi_str, &a, detection_bitmask);
+
 #ifdef CUSTOM_NDPI_PROTOCOLS
 #include "../../../nDPI-custom/custom_ndpi_main_init.c"
 #endif
diff --git a/src/lib/protocols/z3950.c b/src/lib/protocols/z3950.c
new file mode 100644
index 000000000..b267c776d
--- /dev/null
+++ b/src/lib/protocols/z3950.c
@@ -0,0 +1,152 @@
+/*
+ * z3950.c
+ *
+ * Copyright (C) 2012-21 - ntop.org
+ *
+ * This module is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This module is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License.
+ * If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "ndpi_protocol_ids.h"
+
+#define NDPI_CURRENT_PROTO NDPI_PROTOCOL_Z3950
+
+#include <stdlib.h>
+#include "ndpi_api.h"
+
+/* https://github.com/wireshark/wireshark/blob/master/epan/dissectors/asn1/z3950/z3950.asn */
+
+static void ndpi_int_z3950_add_connection(struct ndpi_detection_module_struct *ndpi_struct,
+                                          struct ndpi_flow_struct *flow)
+{
+  ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_Z3950, NDPI_PROTOCOL_UNKNOWN);
+}
+
+static int z3950_parse_sequences(struct ndpi_packet_struct const * const packet,
+                                 int max_sequences)
+{
+  size_t payload_offset = 2;
+  int cur_sequences = 0;
+  u_int8_t pdu_type;
+
+  if (packet->payload_packet_len < 2)
+  {
+    return -1;
+  }
+  pdu_type = packet->payload[0] & 0x1F;
+  if ((pdu_type < 20 || pdu_type > 36) && (pdu_type < 43 || pdu_type > 48))
+  {
+    return -1;
+  }
+
+  while (cur_sequences++ < max_sequences)
+  {
+    if (payload_offset + 2 >= packet->payload_packet_len)
+    {
+      return -1;
+    }
+
+    u_int8_t const * payload;
+    u_int8_t seq_type;
+    u_int8_t seq_length;
+
+    payload = &packet->payload[payload_offset];
+    if (payload[0] == 0x9F)
+    {
+      if (payload_offset + 3 >= packet->payload_packet_len)
+      {
+        return -1;
+      }
+      payload_offset++;
+      payload = &packet->payload[payload_offset];
+      seq_type = payload[0];
+    } else {
+      seq_type = payload[0] & 0x1F;
+    }
+    seq_length = payload[1];
+
+    if (seq_type > 51 && (seq_type < 100 || seq_type > 105) &&
+        (seq_type < 110 || seq_type > 112) && (seq_type < 120 || seq_type > 121) &&
+        (seq_type < 201 || seq_type > 221))
+    {
+      return -1;
+    }
+
+    if (seq_length >= packet->payload_packet_len - payload_offset + 1)
+    {
+      return -1;
+    }
+
+    payload_offset += seq_length + 2;
+
+    if (payload_offset == packet->payload_packet_len)
+    {
+      return cur_sequences;
+    }
+  }
+
+  return cur_sequences - 1;
+}
+
+static void ndpi_search_z3950(struct ndpi_detection_module_struct *ndpi_struct,
+                              struct ndpi_flow_struct *flow)
+{
+  struct ndpi_packet_struct * packet = &flow->packet;
+  int const minimum_expected_sequences = 7;
+
+  NDPI_LOG_DBG(ndpi_struct, "search z39.50\n");
+
+  if (packet->tcp != NULL && packet->payload_packet_len >= 6 &&
+      flow->packet_counter >= 1 && flow->packet_counter <= 8)
+  {
+    int ret = z3950_parse_sequences(packet, minimum_expected_sequences);
+    if (ret < 0)
+    {
+      NDPI_EXCLUDE_PROTO(ndpi_struct, flow);
+      return;
+    }
+    if (ret < minimum_expected_sequences)
+    {
+      /* We've seen not enough sequences, wait for the next packet. */
+      return;
+    }
+
+    if (flow->z3950_stage == 3)
+    {
+      ndpi_int_z3950_add_connection(ndpi_struct, flow);
+    } else {
+      flow->z3950_stage++;
+    }
+    return;
+  }
+
+  NDPI_EXCLUDE_PROTO(ndpi_struct, flow);
+}
+
+
+/* ***************************************************************** */
+
+void init_z3950_dissector(struct ndpi_detection_module_struct *ndpi_struct, u_int32_t *id,
+                          NDPI_PROTOCOL_BITMASK *detection_bitmask)
+{
+  ndpi_set_bitmask_protocol_detection("Z39.50",
+                                      ndpi_struct, detection_bitmask, *id,
+                                      NDPI_PROTOCOL_Z3950,
+                                      ndpi_search_z3950,
+                                      NDPI_SELECTION_BITMASK_PROTOCOL_TCP_WITH_PAYLOAD,
+                                      SAVE_DETECTION_BITMASK_AS_UNKNOWN,
+                                      ADD_TO_DETECTION_BITMASK);
+
+  *id += 1;
+}
diff --git a/tests/pcap/z3950.pcap b/tests/pcap/z3950.pcap
new file mode 100644
index 000000000..713a3acf6
--- /dev/null
+++ b/tests/pcap/z3950.pcap
diff --git a/tests/result/synscan.pcap.out b/tests/result/synscan.pcap.out
index 8e0060550..27812d46f 100644
--- a/tests/result/synscan.pcap.out
+++ b/tests/result/synscan.pcap.out
@@ -97,7 +97,7 @@ iSCSI	2	116	2
 	43	TCP 172.16.0.8:36050 -> 64.13.134.52:2605 [proto: 13/BGP][cat: Network/14][1 pkts/58 bytes -> 0 pkts/0 bytes][Goodput ratio: 0/0][< 1 sec][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
 	44	TCP 172.16.0.8:36050 -> 64.13.134.52:3000 [proto: 26/ntop][cat: Network/14][1 pkts/58 bytes -> 0 pkts/0 bytes][Goodput ratio: 0/0][< 1 sec][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
 	45	TCP 172.16.0.8:36050 -> 64.13.134.52:3128 [proto: 131/HTTP_Proxy][cat: Web/5][1 pkts/58 bytes -> 0 pkts/0 bytes][Goodput ratio: 0/0][< 1 sec][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
-	46	TCP 172.16.0.8:36050 -> 64.13.134.52:3260 [proto: 260/iSCSI][1 pkts/58 bytes -> 0 pkts/0 bytes][Goodput ratio: 0/0][< 1 sec][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
+	46	TCP 172.16.0.8:36050 -> 64.13.134.52:3260 [proto: 261/iSCSI][1 pkts/58 bytes -> 0 pkts/0 bytes][Goodput ratio: 0/0][< 1 sec][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
 	47	TCP 172.16.0.8:36050 -> 64.13.134.52:3306 [proto: 20/MySQL][cat: Database/11][1 pkts/58 bytes -> 0 pkts/0 bytes][Goodput ratio: 0/0][< 1 sec][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
 	48	TCP 172.16.0.8:36050 -> 64.13.134.52:3389 [proto: 88/RDP][cat: RemoteAccess/12][1 pkts/58 bytes -> 0 pkts/0 bytes][Goodput ratio: 0/0][< 1 sec][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
 	49	TCP 172.16.0.8:36050 -> 64.13.134.52:4343 [proto: 170/Whois-DAS][cat: Network/14][1 pkts/58 bytes -> 0 pkts/0 bytes][Goodput ratio: 0/0][< 1 sec][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
@@ -157,7 +157,7 @@ iSCSI	2	116	2
 	103	TCP 172.16.0.8:36051 -> 64.13.134.52:2605 [proto: 13/BGP][cat: Network/14][1 pkts/58 bytes -> 0 pkts/0 bytes][Goodput ratio: 0/0][< 1 sec][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
 	104	TCP 172.16.0.8:36051 -> 64.13.134.52:3000 [proto: 26/ntop][cat: Network/14][1 pkts/58 bytes -> 0 pkts/0 bytes][Goodput ratio: 0/0][< 1 sec][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
 	105	TCP 172.16.0.8:36051 -> 64.13.134.52:3128 [proto: 131/HTTP_Proxy][cat: Web/5][1 pkts/58 bytes -> 0 pkts/0 bytes][Goodput ratio: 0/0][< 1 sec][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
-	106	TCP 172.16.0.8:36051 -> 64.13.134.52:3260 [proto: 260/iSCSI][1 pkts/58 bytes -> 0 pkts/0 bytes][Goodput ratio: 0/0][< 1 sec][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
+	106	TCP 172.16.0.8:36051 -> 64.13.134.52:3260 [proto: 261/iSCSI][1 pkts/58 bytes -> 0 pkts/0 bytes][Goodput ratio: 0/0][< 1 sec][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
 	107	TCP 172.16.0.8:36051 -> 64.13.134.52:3306 [proto: 20/MySQL][cat: Database/11][1 pkts/58 bytes -> 0 pkts/0 bytes][Goodput ratio: 0/0][< 1 sec][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
 	108	TCP 172.16.0.8:36051 -> 64.13.134.52:3389 [proto: 88/RDP][cat: RemoteAccess/12][1 pkts/58 bytes -> 0 pkts/0 bytes][Goodput ratio: 0/0][< 1 sec][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
 	109	TCP 172.16.0.8:36051 -> 64.13.134.52:4343 [proto: 170/Whois-DAS][cat: Network/14][1 pkts/58 bytes -> 0 pkts/0 bytes][Goodput ratio: 0/0][< 1 sec][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
diff --git a/tests/result/z3950.pcap.out b/tests/result/z3950.pcap.out
new file mode 100644
index 000000000..8aed9fd28
--- /dev/null
+++ b/tests/result/z3950.pcap.out
@@ -0,0 +1,3 @@
+Z39.50	15	4997	1
+
+	1	TCP 192.168.2.100:58921 <-> 193.174.240.93:210 [proto: 260/Z39.50][cat: Network/14][7 pkts/623 bytes <-> 8 pkts/4374 bytes][Goodput ratio: 37/90][1.55 sec][bytes ratio: -0.751 (Download)][IAT c2s/s2c min/avg/max/stddev: 0/0 293/29 1341/73 524/28][Pkt Len c2s/s2c min/avg/max/stddev: 54/60 89/547 170/1506 44/623][PLAIN TEXT (p.5.4.1 12b)][Plen Bins: 25,0,25,12,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,12,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,25,0,0]