Merge branch 'dev' of https://github.com/ntop/nDPI into dev-unstable

9 files changed, 522 insertions, 90 deletions

diff --git a/.travis.yml b/.travis.yml
index 57b347449..057e93679 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -76,17 +76,44 @@ matrix:
         - MATRIX_EVAL="CC=clang-7 && CXX=clang++-7"
 
     - name: fuzza
-      env: CXXFLAGS="-DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION -fsanitize=address -fsanitize=fuzzer-no-link" CFLAGS="-DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION -fsanitize=address -fsanitize=fuzzer-no-link" LDFLAGS="-fsanitize=address" QA_FUZZ=asan
-      compiler: clang
+      env: CXXFLAGS="-g -DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION -fsanitize=address -fsanitize=fuzzer-no-link" CFLAGS="-g -DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION -fsanitize=address -fsanitize=fuzzer-no-link" LDFLAGS="-g -fsanitize=address" QA_FUZZ=asan CC=clang-7 && CXX=clang++-7 ASAN_SYMBOLIZER_PATH=/usr/local/clang-7.0.0/bin/llvm-symbolizer
       os: linux
+      compiler: clang-7
+      addons:
+        apt:
+          sources:
+            - ubuntu-toolchain-r-test
+            - llvm-toolchain-trusty-7
+          packages:
+            - clang-7
+            - libpcap-dev
+            - autogen
     - name: fuzzm
-      env: CXXFLAGS="-DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION -fsanitize=memory -fsanitize=fuzzer-no-link" CFLAGS="-DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION -fsanitize=memory -fsanitize=fuzzer-no-link" LDFLAGS="-fsanitize=memory" QA_FUZZ=msan
-      compiler: clang
+      env: CXXFLAGS="-g -DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION -fsanitize=memory -fsanitize=fuzzer-no-link" CFLAGS="-g -DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION -fsanitize=memory -fsanitize=fuzzer-no-link" LDFLAGS="-g -fsanitize=memory" QA_FUZZ=msan CC=clang-7 && CXX=clang++-7 MSAN_SYMBOLIZER_PATH=/usr/local/clang-7.0.0/bin/llvm-symbolizer
       os: linux
+      compiler: clang-7
+      addons:
+        apt:
+          sources:
+            - ubuntu-toolchain-r-test
+            - llvm-toolchain-trusty-7
+          packages:
+            - clang-7
+            - libpcap-dev
+            - autogen
     - name: fuzzu
-      env: CXXFLAGS="-DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION -fsanitize=undefined -fsanitize=fuzzer-no-link" CFLAGS="-DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION -fsanitize=undefined -fno-sanitize-recover=undefined,integer -fsanitize=fuzzer-no-link" LDFLAGS="-fsanitize=undefined" QA_FUZZ=ubsan
-      compiler: clang
+      env: CXXFLAGS="-g -DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION -fsanitize=undefined -fsanitize=fuzzer-no-link" CFLAGS="-g -DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION -fsanitize=undefined -fno-sanitize-recover=undefined,integer -fsanitize=fuzzer-no-link" LDFLAGS="-g -fsanitize=undefined" QA_FUZZ=ubsan CC=clang-7 && CXX=clang++-7
       os: linux
+      compiler: clang-7
+      addons:
+        apt:
+          sources:
+            - ubuntu-toolchain-r-test
+            - llvm-toolchain-trusty-7
+          packages:
+            - clang-7
+            - libpcap-dev
+            - autogen
 
 
   before_install:
diff --git a/configure.seed b/configure.seed
index 326242cd2..68997975e 100644
--- a/configure.seed
+++ b/configure.seed
@@ -160,6 +160,8 @@ AM_CONDITIONAL([HAS_FUZZLDFLAGS], [test "x$has_sanitizefuzzer" = "xyes"])
 
 AC_CHECK_LIB(pthread, pthread_setaffinity_np, AC_DEFINE_UNQUOTED(HAVE_PTHREAD_SETAFFINITY_NP, 1, [libc has pthread_setaffinity_np]))
 
+AC_CHECK_LIB(pcre, pcre_compile, AC_DEFINE_UNQUOTED(HAVE_PCRE, 1, [libpcre(-dev) is present]))
+
 AC_CONFIG_FILES([Makefile example/Makefile example/Makefile.dpdk tests/Makefile libndpi.pc src/include/ndpi_define.h src/lib/Makefile python/Makefile fuzz/Makefile])
 AC_CONFIG_HEADERS(src/include/ndpi_config.h)
 AC_SUBST(GIT_RELEASE)
diff --git a/fuzz/fuzz_ndpi_reader.c b/fuzz/fuzz_ndpi_reader.c
index aee2a0b64..7de3d45f4 100644
--- a/fuzz/fuzz_ndpi_reader.c
+++ b/fuzz/fuzz_ndpi_reader.c
@@ -1,4 +1,5 @@
 #include "reader_util.h"
+#include "ndpi_api.h"
 
 #include <pcap/pcap.h>
 
@@ -17,74 +18,74 @@ u_int8_t human_readeable_string_len = 5;
 u_int8_t max_num_udp_dissected_pkts = 16 /* 8 is enough for most protocols, Signal requires more */, max_num_tcp_dissected_pkts = 80 /* due to telnet */;
 
 int bufferToFile(const char * name, const uint8_t *Data, size_t Size) {
-    FILE * fd;
-    if (remove(name) != 0) {
-        if (errno != ENOENT) {
-            printf("failed remove, errno=%d\n", errno);
-            return -1;
-        }
-    }
-    fd = fopen(name, "wb");
-    if (fd == NULL) {
-        printf("failed open, errno=%d\n", errno);
-        return -2;
-    }
-    if (fwrite (Data, 1, Size, fd) != Size) {
-        fclose(fd);
-        return -3;
+  FILE * fd;
+  if (remove(name) != 0) {
+    if (errno != ENOENT) {
+      printf("failed remove, errno=%d\n", errno);
+      return -1;
     }
+  }
+  fd = fopen(name, "wb");
+  if (fd == NULL) {
+    printf("failed open, errno=%d\n", errno);
+    return -2;
+  }
+  if (fwrite (Data, 1, Size, fd) != Size) {
     fclose(fd);
-    return 0;
+    return -3;
+  }
+  fclose(fd);
+  return 0;
 }
 
 int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) {
-    pcap_t * pkts;
-    const u_char *pkt;
-    struct pcap_pkthdr *header;
-    int r;
-    char errbuf[PCAP_ERRBUF_SIZE];
-    NDPI_PROTOCOL_BITMASK all;
+  pcap_t * pkts;
+  const u_char *pkt;
+  struct pcap_pkthdr *header;
+  int r;
+  char errbuf[PCAP_ERRBUF_SIZE];
+  NDPI_PROTOCOL_BITMASK all;
 
+  if (prefs == NULL) {
+    prefs = calloc(sizeof(struct ndpi_workflow_prefs), 1);
     if (prefs == NULL) {
-        prefs = calloc(sizeof(struct ndpi_workflow_prefs), 1);
-        if (prefs == NULL) {
-            //should not happen
-            return 1;
-        }
-        prefs->decode_tunnels = 1;
-        prefs->num_roots = 16;
-        prefs->max_ndpi_flows = 1024;
-        prefs->quiet_mode = 0;
+      //should not happen
+      return 1;
     }
-    bufferToFile("/tmp/fuzz.pcap", Data, Size);
+    prefs->decode_tunnels = 1;
+    prefs->num_roots = 16;
+    prefs->max_ndpi_flows = 1024;
+    prefs->quiet_mode = 0;
+  }
+  bufferToFile("/tmp/fuzz.pcap", Data, Size);
 
-    pkts = pcap_open_offline("/tmp/fuzz.pcap", errbuf);
-    if (pkts == NULL) {
-        return 0;
-    }
-    struct ndpi_workflow * workflow = ndpi_workflow_init(prefs, pkts);
-    // enable all protocols
-    NDPI_BITMASK_SET_ALL(all);
-    ndpi_set_protocol_detection_bitmask2(workflow->ndpi_struct, &all);
-    memset(workflow->stats.protocol_counter, 0,
-           sizeof(workflow->stats.protocol_counter));
-    memset(workflow->stats.protocol_counter_bytes, 0,
-           sizeof(workflow->stats.protocol_counter_bytes));
-    memset(workflow->stats.protocol_flows, 0,
-           sizeof(workflow->stats.protocol_flows));
-    ndpi_finalize_initalization(workflow->ndpi_struct);
+  pkts = pcap_open_offline("/tmp/fuzz.pcap", errbuf);
+  if (pkts == NULL) {
+    return 0;
+  }
+  struct ndpi_workflow * workflow = ndpi_workflow_init(prefs, pkts);
+  // enable all protocols
+  NDPI_BITMASK_SET_ALL(all);
+  ndpi_set_protocol_detection_bitmask2(workflow->ndpi_struct, &all);
+  memset(workflow->stats.protocol_counter, 0,
+	 sizeof(workflow->stats.protocol_counter));
+  memset(workflow->stats.protocol_counter_bytes, 0,
+	 sizeof(workflow->stats.protocol_counter_bytes));
+  memset(workflow->stats.protocol_flows, 0,
+	 sizeof(workflow->stats.protocol_flows));
+  ndpi_finalize_initalization(workflow->ndpi_struct);
 
+  r = pcap_next_ex(pkts, &header, &pkt);
+  while (r > 0) {
+    /* allocate an exact size buffer to check overflows */
+    uint8_t *packet_checked = malloc(header->caplen);
+    memcpy(packet_checked, pkt, header->caplen);
+    ndpi_workflow_process_packet(workflow, header, packet_checked);
+    free(packet_checked);
     r = pcap_next_ex(pkts, &header, &pkt);
-    while (r > 0) {
-        /* allocate an exact size buffer to check overflows */
-        uint8_t *packet_checked = malloc(header->caplen);
-        memcpy(packet_checked, pkt, header->caplen);
-        ndpi_workflow_process_packet(workflow, header, packet_checked);
-        free(packet_checked);
-        r = pcap_next_ex(pkts, &header, &pkt);
-    }
-    ndpi_workflow_free(workflow);
-    pcap_close(pkts);
+  }
+  ndpi_workflow_free(workflow);
+  pcap_close(pkts);
 
-    return 0;
+  return 0;
 }
diff --git a/src/include/ndpi_api.h b/src/include/ndpi_api.h
index 5723494ba..342a24ae7 100644
--- a/src/include/ndpi_api.h
+++ b/src/include/ndpi_api.h
@@ -857,6 +857,7 @@ extern "C" {
   void ndpi_user_pwd_payload_copy(u_int8_t *dest, u_int dest_len, u_int offset,
 				  const u_int8_t *src, u_int src_len);
   u_char* ndpi_base64_decode(const u_char *src, size_t len, size_t *out_len);
+  char* ndpi_base64_encode(unsigned char const* bytes_to_encode, ssize_t in_len);
   int ndpi_load_ipv4_ptree(struct ndpi_detection_module_struct *ndpi_str,
 			   const char *path, u_int16_t protocol_id);
     
diff --git a/src/lib/ndpi_community_id.c b/src/lib/ndpi_community_id.c
new file mode 100644
index 000000000..7b55fc009
--- /dev/null
+++ b/src/lib/ndpi_community_id.c
@@ -0,0 +1,376 @@
+/*
+ * ndpi_community_id.c
+ *
+ * Copyright (C) 2011-20 - ntop.org
+ *
+ * This file is part of nDPI, an open source deep packet inspection
+ * library based on the OpenDPI and PACE technology by ipoque GmbH
+ *
+ * nDPI is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * nDPI is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with nDPI.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#ifdef HAVE_CONFIG_H
+#include "ndpi_config.h"
+#endif
+
+#include <stdlib.h>
+#include <errno.h>
+#include <sys/types.h>
+
+#include "ndpi_api.h"
+#include "ndpi_config.h"
+
+#include <time.h>
+#ifndef WIN32
+#include <unistd.h>
+#endif
+
+#if defined __FreeBSD__ || defined __NetBSD__ || defined __OpenBSD__
+#include <sys/endian.h>
+#endif
+
+#include "ndpi_sha1.h"
+
+#define NDPI_ICMP6_ECHO_REQUEST		128
+#define NDPI_ICMP6_ECHO_REPLY		129
+#define NDPI_MLD_LISTENER_QUERY		130
+#define NDPI_MLD_LISTENER_REPORT	131
+
+#define NDPI_ROUTER_SOLICIT		133
+#define NDPI_ROUTER_ADVERT		134
+#define NDPI_NEIGHBOR_SOLICIT		135
+#define NDPI_NEIGHBOR_ADVERT		136
+
+#define NDPI_ICMP_ECHOREPLY		0
+#define	NDPI_ICMP_ECHO			8
+#define	NDPI_ICMP_ROUTERADVERT		9
+#define	NDPI_ICMP_ROUTERSOLICIT		10
+#define	NDPI_ICMP_TIMESTAMP		13
+#define	NDPI_ICMP_TIMESTAMPREPLY	14
+#define	NDPI_ICMP_INFO_REQUEST		15
+#define	NDPI_ICMP_INFO_REPLY		16
+#define	NDPI_ICMP_MASKREQ		17
+#define	NDPI_ICMP_MASKREPLY		18
+
+#define NDPI_ICMP6_WRUREQUEST		139
+#define NDPI_ICMP6_WRUREPLY		140
+
+/* **************************************************** */
+
+static ssize_t ndpi_community_id_buf_copy(u_int8_t * const dst, const void * const src, ssize_t len) {
+  if(src)
+    memcpy(dst, src, len);
+  else
+    memset(dst, 0, len);
+
+  return len;
+}
+
+/* **************************************************** */
+
+/*
+  https://github.com/corelight/community-id-spec/blob/bda913f617389df07cdaa23606e11bbd318e265c/community-id.py#L56
+*/
+static u_int8_t ndpi_community_id_icmp_type_to_code_v4(u_int8_t icmp_type, u_int8_t icmp_code, int *is_one_way) {
+  *is_one_way = 0;
+
+  switch(icmp_type) {
+  case NDPI_ICMP_ECHO:
+    return NDPI_ICMP_ECHOREPLY;
+  case NDPI_ICMP_ECHOREPLY:
+    return NDPI_ICMP_ECHO;
+  case NDPI_ICMP_TIMESTAMP:
+    return NDPI_ICMP_TIMESTAMPREPLY;
+  case NDPI_ICMP_TIMESTAMPREPLY:
+    return NDPI_ICMP_TIMESTAMP;
+  case NDPI_ICMP_INFO_REQUEST:
+    return NDPI_ICMP_INFO_REPLY;
+  case NDPI_ICMP_INFO_REPLY:
+    return NDPI_ICMP_INFO_REQUEST;
+  case NDPI_ICMP_ROUTERSOLICIT:
+    return NDPI_ICMP_ROUTERADVERT;
+  case NDPI_ICMP_ROUTERADVERT:
+    return NDPI_ICMP_ROUTERSOLICIT;
+  case NDPI_ICMP_MASKREQ:
+    return NDPI_ICMP_MASKREPLY;
+  case NDPI_ICMP_MASKREPLY:
+    return NDPI_ICMP_MASKREQ;
+  default:
+    *is_one_way = 1;
+    return icmp_code;
+  }
+}
+
+/* **************************************************** */
+
+/*
+  https://github.com/corelight/community-id-spec/blob/bda913f617389df07cdaa23606e11bbd318e265c/community-id.py#L83
+*/
+static u_int8_t ndpi_community_id_icmp_type_to_code_v6(u_int8_t icmp_type, u_int8_t icmp_code, int *is_one_way) {
+  *is_one_way = 0;
+
+  switch(icmp_type) {
+  case NDPI_ICMP6_ECHO_REQUEST:
+    return NDPI_ICMP6_ECHO_REPLY;
+  case NDPI_ICMP6_ECHO_REPLY:
+    return NDPI_ICMP6_ECHO_REQUEST;
+  case NDPI_ROUTER_SOLICIT:
+    return NDPI_ROUTER_ADVERT;
+  case NDPI_ROUTER_ADVERT:
+    return NDPI_ROUTER_SOLICIT;
+  case NDPI_NEIGHBOR_SOLICIT:
+    return NDPI_NEIGHBOR_ADVERT;
+  case NDPI_NEIGHBOR_ADVERT:
+    return NDPI_NEIGHBOR_SOLICIT;
+  case NDPI_MLD_LISTENER_QUERY:
+    return NDPI_MLD_LISTENER_REPORT;
+  case NDPI_MLD_LISTENER_REPORT:
+    return NDPI_MLD_LISTENER_QUERY;
+  case NDPI_ICMP6_WRUREQUEST:
+    return NDPI_ICMP6_WRUREPLY;
+  case NDPI_ICMP6_WRUREPLY:
+    return NDPI_ICMP6_WRUREQUEST;
+  // Home Agent Address Discovery Request Message and reply
+  case 144:
+    return 145;
+  case 145:
+    return 144;
+  default:
+    *is_one_way = 1;
+    return icmp_code;
+  }
+}
+
+/* **************************************************** */
+
+/* 
+  https://github.com/corelight/community-id-spec/blob/bda913f617389df07cdaa23606e11bbd318e265c/community-id.py#L164
+*/
+static int ndpi_community_id_peer_v4_is_less_than(u_int32_t ip1, u_int32_t ip2, u_int16_t p1, u_int16_t p2) {
+  int comp = memcmp(&ip1, &ip2, sizeof(u_int32_t));
+  return comp < 0 || (comp == 0 && p1 < p2);
+}
+
+/* **************************************************** */
+
+static int ndpi_community_id_peer_v6_is_less_than(struct ndpi_in6_addr *ip1, struct ndpi_in6_addr *ip2, u_int16_t p1, u_int16_t p2) {
+  int comp = memcmp(ip1, ip2, sizeof(struct ndpi_in6_addr));
+  return comp < 0 || (comp == 0 && p1 < p2);
+}
+
+/* **************************************************** */
+
+static void ndpi_community_id_sha1_hash(const uint8_t *message, size_t len, u_char *hash /* 20-bytes */) {
+  SHA1_CTX ctx = { 0 };
+  SHA1Init(&ctx);
+  SHA1Update(&ctx, message, len);
+  SHA1Final(hash, &ctx);
+}
+
+/* **************************************************** */
+
+/*
+https://github.com/corelight/community-id-spec/blob/bda913f617389df07cdaa23606e11bbd318e265c/community-id.py#L285
+*/
+static int ndpi_community_id_finalize_and_compute_hash(u_int8_t *comm_buf, u_int16_t off, u_int8_t l4_proto,
+             u_int16_t src_port, u_int16_t dst_port, char *hash_buf, u_int8_t hash_buf_len) {
+  u_int8_t pad = 0;
+  uint32_t hash[5];
+  char *community_id;
+
+  /* L4 proto */
+  off += ndpi_community_id_buf_copy(&comm_buf[off], &l4_proto, sizeof(l4_proto));
+
+  /* Pad */
+  off += ndpi_community_id_buf_copy(&comm_buf[off], &pad, sizeof(pad));
+
+  /* Source and destination ports */
+  switch(l4_proto) {
+  case IPPROTO_ICMP:
+  case IPPROTO_ICMPV6:
+  case IPPROTO_SCTP:
+  case IPPROTO_UDP:
+  case IPPROTO_TCP:
+    off += ndpi_community_id_buf_copy(&comm_buf[off], &src_port, sizeof(src_port));
+    off += ndpi_community_id_buf_copy(&comm_buf[off], &dst_port, sizeof(dst_port));
+    break;
+  }
+
+  /* Compute SHA1 */
+  ndpi_community_id_sha1_hash(comm_buf, off, (u_char*)hash);
+
+  /* Base64 encoding */
+  community_id = ndpi_base64_encode((u_int8_t*)hash, sizeof(hash));
+
+  if (community_id == NULL)
+    return -1;
+
+#if 1 /* Debug Info */
+  printf("Hex output: ");
+  for(int i = 0; i < off; i++)
+    printf("%.2x ", comm_buf[i]);
+  printf("\n");
+
+  printf("Sha1 sum: ");
+  for(int i = 0; i < 5; i++)
+    printf("%.2x ", ntohl(hash[i]));
+  printf("\n");
+
+  printf("Base64: %s\n", community_id);
+#endif
+
+  if (hash_buf_len < 2 || hash_buf_len-2 < strlen(community_id)+1) {
+    ndpi_free(community_id);
+    return -1;
+  }
+
+  /* Writing hash */
+  hash_buf[0] = '1';
+  hash_buf[1] = ':';
+  strcpy(&hash_buf[2], community_id);
+  ndpi_free(community_id);
+
+  return 0;
+}
+
+/* **************************************************** */
+
+/*
+  NOTE:
+  - Leave fields empty/zero when information is missing (e.g. with ICMP ports are zero)
+  - The hash_buf most be 30+1 bits or longer
+  - Return code: 0 = OK, -1 otherwise
+*/
+
+int ndpi_flowv4_flow_hash(u_int8_t l4_proto, u_int32_t src_ip, u_int32_t dst_ip,
+                          u_int16_t src_port, u_int16_t dst_port,
+                          u_int8_t icmp_type, u_int8_t icmp_code,
+                          u_char *hash_buf, u_int8_t hash_buf_len) {
+  /*
+    Input buffer (40 bytes)
+     2 - Seed 
+    16 - IPv6 src 
+    16 - IPv6 dst
+     1 - L4 proto
+     1 - Pad 
+     2 - Port src
+     2 - Port dst
+  */
+  u_int8_t comm_buf[40] = { 0 };
+  u_int16_t off = 0;
+  u_int16_t seed = 0;
+  u_int32_t *ip_a_ptr, *ip_b_ptr;
+  u_int16_t port_a, port_b;
+  int icmp_one_way = 0;
+
+  /* Adjust the ports according to the specs */
+  switch(l4_proto) {
+  case IPPROTO_ICMP:
+    src_port = icmp_type;
+    dst_port = ndpi_community_id_icmp_type_to_code_v4(icmp_type, icmp_code, &icmp_one_way);
+    break;
+  case IPPROTO_SCTP:
+  case IPPROTO_UDP:
+  case IPPROTO_TCP:
+    /* src/dst port ok */
+    break;
+  default:
+    src_port = dst_port = 0;
+    break;
+  }
+
+  /* Convert tuple to NBO */
+  src_ip = htonl(src_ip);
+  dst_ip = htonl(dst_ip);
+  src_port = htons(src_port);
+  dst_port = htons(dst_port);
+
+  /*
+    The community id hash doesn't have the definition of client and server, it just sorts IP addresses
+    and ports to make sure the smaller ip address is the first. This performs this check and
+    possibly swap client ip and port.
+  */
+  if(icmp_one_way || ndpi_community_id_peer_v4_is_less_than(src_ip, dst_ip, src_port, dst_port)) {
+    ip_a_ptr = &src_ip, ip_b_ptr = &dst_ip;
+    port_a = src_port, port_b = dst_port;
+  } else {
+    /* swap flow peers */
+    ip_a_ptr = &dst_ip, ip_b_ptr = &src_ip;
+    port_a = dst_port, port_b = src_port;
+  }
+
+  /* Seed */
+  off = ndpi_community_id_buf_copy(&comm_buf[off], &seed, sizeof(seed));
+
+  /* Source and destination IPs */
+  off += ndpi_community_id_buf_copy(&comm_buf[off], ip_a_ptr, sizeof(src_ip));
+  off += ndpi_community_id_buf_copy(&comm_buf[off], ip_b_ptr, sizeof(dst_ip));
+
+  return ndpi_community_id_finalize_and_compute_hash(comm_buf, off,
+           l4_proto, port_a, port_b, (char*)hash_buf, hash_buf_len);
+}
+
+/* **************************************************** */
+
+int ndpi_flowv6_flow_hash(u_int8_t l4_proto, struct ndpi_in6_addr *src_ip, struct ndpi_in6_addr *dst_ip,
+                          u_int16_t src_port, u_int16_t dst_port,
+                          u_int8_t icmp_type, u_int8_t icmp_code,
+                          u_char *hash_buf, u_int8_t hash_buf_len) {
+  u_int8_t comm_buf[40] = { 0 };
+  u_int16_t off = 0;
+  u_int16_t seed = 0;
+  struct ndpi_in6_addr *ip_a_ptr, *ip_b_ptr;
+  u_int16_t port_a, port_b;
+  int icmp_one_way = 0;
+
+  switch(l4_proto) {
+  case IPPROTO_ICMPV6:
+    src_port = icmp_type;
+    dst_port = ndpi_community_id_icmp_type_to_code_v6(icmp_type, icmp_code, &icmp_one_way);
+    break;
+  case IPPROTO_SCTP:
+  case IPPROTO_UDP:
+  case IPPROTO_TCP:
+    /* src/dst port ok */
+    break;
+  default:
+    src_port = dst_port = 0;
+    break;
+  }
+
+  /* Convert tuple to NBO */
+  src_port = htons(src_port);
+  dst_port = htons(dst_port);
+
+  if(icmp_one_way || ndpi_community_id_peer_v6_is_less_than(src_ip, dst_ip, src_port, dst_port)) {
+    ip_a_ptr = src_ip, ip_b_ptr = dst_ip;
+    port_a = src_port, port_b = dst_port;
+  } else {
+    ip_a_ptr = dst_ip, ip_b_ptr = src_ip;
+    port_a = dst_port, port_b = src_port;
+  }
+
+  /* Seed */
+  off = ndpi_community_id_buf_copy(&comm_buf[off], &seed, sizeof(seed));
+
+  /* Source and destination IPs */
+  off += ndpi_community_id_buf_copy(&comm_buf[off], ip_a_ptr, sizeof(struct ndpi_in6_addr));
+  off += ndpi_community_id_buf_copy(&comm_buf[off], ip_b_ptr, sizeof(struct ndpi_in6_addr));
+
+  return ndpi_community_id_finalize_and_compute_hash(comm_buf, off,
+           l4_proto, port_a, port_b, (char*)hash_buf, hash_buf_len);
+}
+
+/* **************************************************** */
diff --git a/src/lib/ndpi_main.c b/src/lib/ndpi_main.c
index 90c9a81a4..9c406d988 100644
--- a/src/lib/ndpi_main.c
+++ b/src/lib/ndpi_main.c
@@ -6569,30 +6569,6 @@ void ndpi_lru_add_to_cache(struct ndpi_lru_cache *c, u_int32_t key, u_int16_t va
 /* ******************************************************************** */
 
 /*
-  NOTE:
-  - Leave fields empty/zero when information is missing (e.g. with ICMP ports are zero)
-  - The hash_buf most be 30+1 bits or longer
-  - Return code: 0 = OK, -1 otherwise
-*/
-
-int ndpi_flowv4_flow_hash(u_int8_t l4_proto, u_int32_t src_ip,
-			  u_int32_t dst_ip, u_int16_t src_port, u_int16_t dst_port,
-			  u_int8_t icmp_type, u_int8_t icmp_code,
-			  u_char *hash_buf, u_int8_t hash_buf_len) {
-
-  return(0); /* OK */
-}
-
-int ndpi_flowv6_flow_hash(u_int8_t l4_proto, struct ndpi_in6_addr *src_ip, struct ndpi_in6_addr *dst_ip,
-			  u_int16_t src_port, u_int16_t dst_port, u_int8_t icmp_type, u_int8_t icmp_code,
-			  u_char *hash_buf, u_int8_t hash_buf_len) {
-
-  return(0); /* OK */
-}
-
-/* ******************************************************************** */
-
-/*
    This function tells if it's possible to further dissect a given flow
    0 - All possible dissection has been completed
    1 - Additional dissection is possible
diff --git a/src/lib/ndpi_utils.c b/src/lib/ndpi_utils.c
index c6844ffa3..c7ea25aee 100644
--- a/src/lib/ndpi_utils.c
+++ b/src/lib/ndpi_utils.c
@@ -853,6 +853,55 @@ u_char* ndpi_base64_decode(const u_char *src, size_t len, size_t *out_len) {
 }
 
 /* ********************************** */
+
+char* ndpi_base64_encode(unsigned char const* bytes_to_encode, ssize_t in_len) {
+  ssize_t len = 0, ret_size;
+  char *ret;
+  int i = 0;
+  unsigned char char_array_3[3];
+  unsigned char char_array_4[4];
+
+  ret_size = ((in_len+2)/3)*4;
+
+  if((ret = (char*)ndpi_malloc(ret_size+1)) == NULL)
+    return NULL;
+
+  while (in_len--) {
+    char_array_3[i++] = *(bytes_to_encode++);
+    if(i == 3) {
+      char_array_4[0] = (char_array_3[0] & 0xfc) >> 2;
+      char_array_4[1] = ((char_array_3[0] & 0x03) << 4) + ((char_array_3[1] & 0xf0) >> 4);
+      char_array_4[2] = ((char_array_3[1] & 0x0f) << 2) + ((char_array_3[2] & 0xc0) >> 6);
+      char_array_4[3] = char_array_3[2] & 0x3f;
+
+      for(i = 0; i < 4; i++)
+        ret[len++] = base64_table[char_array_4[i]];
+      i = 0;
+    }
+  }
+
+  if(i) {
+    for(int j = i; j < 3; j++)
+      char_array_3[j] = '\0';
+
+    char_array_4[0] = (char_array_3[0] & 0xfc) >> 2;
+    char_array_4[1] = ((char_array_3[0] & 0x03) << 4) + ((char_array_3[1] & 0xf0) >> 4);
+    char_array_4[2] = ((char_array_3[1] & 0x0f) << 2) + ((char_array_3[2] & 0xc0) >> 6);
+    char_array_4[3] = char_array_3[2] & 0x3f;
+
+    for(int j = 0; (j < i + 1); j++)
+      ret[len++] = base64_table[char_array_4[j]];
+
+    while((i++ < 3))
+      ret[len++] = '=';
+  }
+
+  ret[len++] = '\0';
+
+  return ret;
+}
+
+/* ********************************** */
 /* ********************************** */
 
 int ndpi_flow2json(struct ndpi_detection_module_struct *ndpi_struct,
diff --git a/src/lib/protocols/citrix.c b/src/lib/protocols/citrix.c
index a05d748ca..65852da7f 100644
--- a/src/lib/protocols/citrix.c
+++ b/src/lib/protocols/citrix.c
@@ -51,7 +51,7 @@ static void ndpi_check_citrix(struct ndpi_detection_module_struct *ndpi_struct,
 	  ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_CITRIX, NDPI_PROTOCOL_UNKNOWN);
 	}
 	return;
-      } else if(payload_len > 4) {
+      } else if(payload_len > 22) {
 	char citrix_header[] = { 0x1a, 0x43, 0x47, 0x50, 0x2f, 0x30, 0x31 };
 	
 	if((memcmp(packet->payload, citrix_header, sizeof(citrix_header)) == 0)
diff --git a/src/lib/protocols/http.c b/src/lib/protocols/http.c
index 14be88246..00522cfb9 100644
--- a/src/lib/protocols/http.c
+++ b/src/lib/protocols/http.c
@@ -300,7 +300,7 @@ static void check_content_type_and_change_protocol(struct ndpi_detection_module_
 	      setHttpUserAgent(ndpi_struct, flow, token);
 	  }
 	}
-      } else if(memcmp(ua, "netflix-ios-app", 15) == 0) {
+      } else if((packet->user_agent_line.len > 14) && (memcmp(ua, "netflix-ios-app", 15) == 0)) {
 	NDPI_LOG_INFO(ndpi_struct, "found netflix\n");
       	ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_NETFLIX, NDPI_PROTOCOL_CATEGORY_STREAMING);
       	return;