Renamed ndpi_util.c -> reader_util.c

Added crash fix with IPv6

1 files changed, 251 insertions, 0 deletions

diff --git a/example/reader_util.h b/example/reader_util.h
new file mode 100644
index 000000000..b006fd8d3
--- /dev/null
+++ b/example/reader_util.h
@@ -0,0 +1,251 @@
+/*
+ * ndpi_util.h
+ *
+ * Copyright (C) 2011-18 - ntop.org
+ *
+ * nDPI is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * nDPI is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with nDPI.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+/**
+ * This module contains routines to help setup a simple nDPI program.
+ *
+ * If you concern about performance or have to integrate nDPI in your
+ * application, you could need to reimplement them yourself.
+ *
+ * WARNING: this API is just a demo od nDPI usage: Use it at your own risk!
+ */
+#ifndef __NDPI_UTIL_H__
+#define __NDPI_UTIL_H__
+
+#include "uthash.h"
+#include <pcap.h>
+
+#ifdef USE_DPDK
+#include <rte_eal.h>
+#include <rte_ether.h>
+#include <rte_ethdev.h>
+#include <rte_cycles.h>
+#include <rte_lcore.h>
+#include <rte_mbuf.h>
+
+#define RX_RING_SIZE     128
+#define TX_RING_SIZE     512
+#define NUM_MBUFS       8191
+#define MBUF_CACHE_SIZE  250
+#define BURST_SIZE        32
+#define PREFETCH_OFFSET    3
+
+extern int dpdk_port_init(int port, struct rte_mempool *mbuf_pool);
+#endif
+
+#define MAX_NUM_READER_THREADS     16
+#define IDLE_SCAN_PERIOD           10 /* msec (use TICK_RESOLUTION = 1000) */
+#define MAX_IDLE_TIME           30000
+#define IDLE_SCAN_BUDGET         1024
+#define NUM_ROOTS                 512
+#define MAX_EXTRA_PACKETS_TO_CHECK  7
+#define MAX_NDPI_FLOWS      200000000
+#define TICK_RESOLUTION          1000
+#define MAX_NUM_IP_ADDRESS          5  /* len of ip address array */
+#define UPDATED_TREE                1
+#define AGGRESSIVE_PERCENT      95.00
+#define DIR_SRC                    10
+#define DIR_DST                    20
+#define PORT_ARRAY_SIZE            20
+#define HOST_ARRAY_SIZE            20
+#define FLOWS_PACKETS_THRESHOLD   0.9
+#define FLOWS_PERCENT_THRESHOLD   1.0
+#define FLOWS_PERCENT_THRESHOLD_2 0.2
+#define FLOWS_THRESHOLD          1000
+#define PKTS_PERCENT_THRESHOLD    0.1
+#define MAX_TABLE_SIZE_1         4096
+#define MAX_TABLE_SIZE_2         8192
+#define INIT_VAL                   -1
+
+
+// inner hash table (ja3 -> security state)
+typedef struct ndpi_ja3_info {
+   char * ja3;
+   ndpi_cipher_weakness unsafe_cipher;
+   UT_hash_handle hh;
+} ndpi_ja3_info;
+
+// external hash table (host ip -> <ip string, hash table ja3c, hash table ja3s>)
+// used to aggregate ja3 fingerprints by hosts
+typedef struct ndpi_host_ja3_fingerprints{
+   u_int32_t ip;
+   char *ip_string;
+   char *dns_name;
+   ndpi_ja3_info *host_client_info_hasht;
+   ndpi_ja3_info *host_server_info_hasht;
+
+   UT_hash_handle hh;
+} ndpi_host_ja3_fingerprints;
+
+
+//inner hash table
+typedef struct ndpi_ip_dns{
+   u_int32_t ip;
+   char *ip_string;
+   char *dns_name; //server name if any;
+   UT_hash_handle hh;
+} ndpi_ip_dns;
+
+//hash table ja3 -> <host, ip, security>, used to aggregate host by ja3 fingerprints
+typedef struct ndpi_ja3_fingerprints_host{
+   char *ja3; //key
+   ndpi_cipher_weakness unsafe_cipher;
+   ndpi_ip_dns *ipToDNS_ht;
+   UT_hash_handle hh;
+} ndpi_ja3_fingerprints_host;
+
+
+
+// flow tracking
+typedef struct ndpi_flow_info {
+  u_int32_t hashval;
+  u_int32_t src_ip;
+  u_int32_t dst_ip;
+  u_int16_t src_port;
+  u_int16_t dst_port;
+  u_int8_t detection_completed, protocol, bidirectional, check_extra_packets;
+  u_int16_t vlan_id;
+  struct ndpi_flow_struct *ndpi_flow;
+  char src_name[48], dst_name[48];
+  u_int8_t ip_version;
+  u_int64_t last_seen;
+  u_int64_t src2dst_bytes, dst2src_bytes;
+  u_int32_t src2dst_packets, dst2src_packets;
+  u_int32_t has_human_readeable_strings;
+  char human_readeable_string_buffer[32];
+  
+  // result only, not used for flow identification
+  ndpi_protocol detected_protocol;
+
+  char info[96];
+  char host_server_name[256];
+  char bittorent_hash[41];
+  char dhcp_fingerprint[48];
+
+  struct {
+    u_int16_t ssl_version;
+    char client_info[64], server_info[64], server_organization[64],
+      ja3_client[33], ja3_server[33];
+    u_int16_t server_cipher;
+    ndpi_cipher_weakness client_unsafe_cipher, server_unsafe_cipher;
+  } ssh_ssl;
+
+  void *src_id, *dst_id;
+} ndpi_flow_info_t;
+
+
+// flow statistics info
+typedef struct ndpi_stats {
+  u_int32_t guessed_flow_protocols;
+  u_int64_t raw_packet_count;
+  u_int64_t ip_packet_count;
+  u_int64_t total_wire_bytes, total_ip_bytes, total_discarded_bytes;
+  u_int64_t protocol_counter[NDPI_MAX_SUPPORTED_PROTOCOLS + NDPI_MAX_NUM_CUSTOM_PROTOCOLS + 1];
+  u_int64_t protocol_counter_bytes[NDPI_MAX_SUPPORTED_PROTOCOLS + NDPI_MAX_NUM_CUSTOM_PROTOCOLS + 1];
+  u_int32_t protocol_flows[NDPI_MAX_SUPPORTED_PROTOCOLS + NDPI_MAX_NUM_CUSTOM_PROTOCOLS + 1];
+  u_int32_t ndpi_flow_count;
+  u_int64_t tcp_count, udp_count;
+  u_int64_t mpls_count, pppoe_count, vlan_count, fragmented_count;
+  u_int64_t packet_len[6];
+  u_int16_t max_packet_len;
+} ndpi_stats_t;
+
+
+// flow preferences
+typedef struct ndpi_workflow_prefs {
+  u_int8_t decode_tunnels;
+  u_int8_t quiet_mode;
+  u_int32_t num_roots;
+  u_int32_t max_ndpi_flows;
+} ndpi_workflow_prefs_t;
+
+struct ndpi_workflow;
+
+/** workflow, flow, user data */
+typedef void (*ndpi_workflow_callback_ptr) (struct ndpi_workflow *, struct ndpi_flow_info *, void *);
+
+
+// workflow main structure
+typedef struct ndpi_workflow {
+  u_int64_t last_time;
+
+  struct ndpi_workflow_prefs prefs;
+  struct ndpi_stats stats;
+
+  ndpi_workflow_callback_ptr __flow_detected_callback;
+  void * __flow_detected_udata;
+  ndpi_workflow_callback_ptr __flow_giveup_callback;
+  void * __flow_giveup_udata;
+
+  /* outside referencies */
+  pcap_t *pcap_handle;
+
+  /* allocated by prefs */
+  void **ndpi_flows_root;
+  struct ndpi_detection_module_struct *ndpi_struct;
+  u_int32_t num_allocated_flows;
+ } ndpi_workflow_t;
+
+
+/* TODO: remove wrappers parameters and use ndpi global, when their initialization will be fixed... */
+struct ndpi_workflow * ndpi_workflow_init(const struct ndpi_workflow_prefs * prefs, pcap_t * pcap_handle);
+
+
+ /* workflow main free function */
+void ndpi_workflow_free(struct ndpi_workflow * workflow);
+
+
+/** Free flow_info ndpi support structures but not the flow_info itself
+ *
+ *  TODO remove! Half freeing things is bad!
+ */
+void ndpi_free_flow_info_half(struct ndpi_flow_info *flow);
+
+
+/* Process a packet and update the workflow  */
+struct ndpi_proto ndpi_workflow_process_packet(struct ndpi_workflow * workflow,
+					       const struct pcap_pkthdr *header,
+					       const u_char *packet);
+
+
+/* flow callbacks for complete detected flow
+   (ndpi_flow_info will be freed right after) */
+static inline void ndpi_workflow_set_flow_detected_callback(struct ndpi_workflow * workflow, ndpi_workflow_callback_ptr callback, void * udata) {
+  workflow->__flow_detected_callback = callback;
+  workflow->__flow_detected_udata = udata;
+}
+
+/* flow callbacks for sufficient detected flow
+   (ndpi_flow_info will be freed right after) */
+static inline void ndpi_workflow_set_flow_giveup_callback(struct ndpi_workflow * workflow, ndpi_workflow_callback_ptr callback, void * udata) {
+  workflow->__flow_giveup_callback = callback;
+  workflow->__flow_giveup_udata = udata;
+}
+
+ /* compare two nodes in workflow */
+int ndpi_workflow_node_cmp(const void *a, const void *b);
+void process_ndpi_collected_info(struct ndpi_workflow * workflow, struct ndpi_flow_info *flow);
+u_int32_t ethernet_crc32(const void* data, size_t n_bytes);
+void ndpi_flow_info_freer(void *node);
+const char* print_cipher_id(u_int32_t cipher);
+
+extern int nDPI_LogLevel;
+
+#endif