Add unit tests to travis. Move ndpi serializer tests to unit tests.

6 files changed, 303 insertions, 152 deletions

diff --git a/.travis.yml b/.travis.yml
index cbe38fc05..5a91176ff 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -13,6 +13,7 @@ matrix:
           packages:
             - libpcap-dev
             - libgcrypt20-dev
+            - libjson-c-dev
             - autogen
 
     - os: linux
@@ -23,6 +24,7 @@ matrix:
           packages:
             - libpcap-dev
             - libgcrypt20-dev
+            - libjson-c-dev
             - autogen
 
     - os: linux
@@ -34,6 +36,7 @@ matrix:
           packages:
             - libpcap-dev
             - libgcrypt20-dev
+            - libjson-c-dev
             - autogen
 
     - os: linux
@@ -45,6 +48,7 @@ matrix:
           packages:
             - libpcap-dev
             - libgcrypt20-dev
+            - libjson-c-dev
             - autogen
 
 # Targets below have been disabled as we have no way
@@ -57,6 +61,7 @@ matrix:
 #        apt:
 #          packages:
 #            - libpcap-dev
+#            - libjson-c-dev
 #            - autogen
 #
 #    - os: linux
@@ -67,6 +72,7 @@ matrix:
 #        apt:
 #          packages:
 #            - libpcap-dev
+#            - libjson-c-dev
 #            - autogen
 
     - os: linux
@@ -79,6 +85,7 @@ matrix:
             - g++-8
             - libpcap-dev
             - libgcrypt20-dev
+            - libjson-c-dev
             - autogen
       env:
         - MATRIX_EVAL="CC=gcc-8"
@@ -93,6 +100,7 @@ matrix:
             - g++-9
             - libpcap-dev
             - libgcrypt20-dev
+            - libjson-c-dev
             - autogen
       env:
         - MATRIX_EVAL="CC=gcc-9"
@@ -108,6 +116,7 @@ matrix:
             - clang-8
             - libpcap-dev
             - libgcrypt20-dev
+            - libjson-c-dev
             - autogen
       env:
         - MATRIX_EVAL="CC=clang-8"
@@ -123,6 +132,7 @@ matrix:
             - clang-7
             - libpcap-dev
             - libgcrypt20-dev
+            - libjson-c-dev
             - autogen
       env:
         - MATRIX_EVAL="CC=clang-7"
@@ -140,6 +150,7 @@ matrix:
             - clang-7
             - libpcap-dev
             - libgcrypt20-dev
+            - libjson-c-dev
             - autogen
     - name: fuzzm
       env: CXXFLAGS="-g3 -O0 -DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION -fsanitize=memory -fsanitize=fuzzer-no-link" CFLAGS="-g3 -O0 -DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION -fsanitize=memory -fsanitize=fuzzer-no-link" LDFLAGS="-g3 -O0 -fsanitize=memory" QA_FUZZ=msan CC=clang && CXX=clang++
@@ -151,6 +162,7 @@ matrix:
           packages:
             - libpcap-dev
             - libgcrypt20-dev
+            - libjson-c-dev
             - autogen
     - name: fuzzu
       env: CXXFLAGS="-g3 -O0 -DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION -fsanitize=undefined -fsanitize=fuzzer-no-link" CFLAGS="-g3 -O0 -DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION -fsanitize=undefined -fno-sanitize-recover=undefined,integer -fsanitize=fuzzer-no-link" LDFLAGS="-g3 -O0 -fsanitize=undefined" QA_FUZZ=ubsan CC=clang-7 && CXX=clang++-7
@@ -165,6 +177,7 @@ matrix:
             - clang-7
             - libpcap-dev
             - libgcrypt20-dev
+            - libjson-c-dev
             - autogen
 
 
diff --git a/configure.seed b/configure.seed
index 37a269943..b8ec621af 100644
--- a/configure.seed
+++ b/configure.seed
@@ -5,7 +5,7 @@ AC_CONFIG_MACRO_DIR([m4])
 
 AM_INIT_AUTOMAKE([foreign subdir-objects])
 
-EXTRA_TARGETS="example tests"
+EXTRA_TARGETS="example tests tests/unit"
 AC_ARG_WITH(only-libndpi,  AS_HELP_STRING([--with-only-libndpi], [Build only libndpi (no examples, tests etc)]))
 AS_IF([test "${with_only_libndpi+set}" = set],[
   EXTRA_TARGETS=""
@@ -189,7 +189,7 @@ if test "${with_pcre+set}" = set; then :
 fi
 
 
-AC_CONFIG_FILES([Makefile example/Makefile example/Makefile.dpdk tests/Makefile libndpi.pc src/include/ndpi_define.h src/lib/Makefile python/Makefile fuzz/Makefile src/include/ndpi_api.h])
+AC_CONFIG_FILES([Makefile example/Makefile example/Makefile.dpdk tests/Makefile tests/unit/Makefile libndpi.pc src/include/ndpi_define.h src/lib/Makefile python/Makefile fuzz/Makefile src/include/ndpi_api.h])
 AC_CONFIG_HEADERS(src/include/ndpi_config.h)
 AC_SUBST(GIT_RELEASE)
 AC_SUBST(NDPI_MAJOR)
diff --git a/example/ndpiReader.c b/example/ndpiReader.c
index e5609f3b0..7f3fa08e5 100644
--- a/example/ndpiReader.c
+++ b/example/ndpiReader.c
@@ -3386,155 +3386,6 @@ void automataUnitTest() {
 
 /* *********************************************** */
 
-void serializerUnitTest() {
-  ndpi_serializer serializer, deserializer;
-  int i, loop_id;
-  u_int8_t trace = 0;
-  ndpi_serialization_format fmt;
-
-  for(loop_id=0; loop_id<3; loop_id++) {
-    switch(loop_id) {
-    case 0:
-      if (trace) printf("--- TLV test ---\n");
-      fmt = ndpi_serialization_format_tlv;
-      break;
-
-    case 1:
-      if (trace) printf("--- JSON test ---\n");
-      fmt = ndpi_serialization_format_json;
-      break;
-
-    case 2:
-      if (trace) printf("--- CSV test ---\n");
-      fmt = ndpi_serialization_format_csv;
-      break;
-    }
-    assert(ndpi_init_serializer(&serializer, fmt) != -1);
-
-    for(i=0; i<16; i++) {
-      char kbuf[32], vbuf[32];
-      snprintf(kbuf, sizeof(kbuf), "Key %d", i);
-      snprintf(vbuf, sizeof(vbuf), "Value %d", i);
-      assert(ndpi_serialize_uint32_uint32(&serializer, i, i*i) != -1);
-      assert(ndpi_serialize_uint32_string(&serializer, i, "Data") != -1);
-      assert(ndpi_serialize_string_string(&serializer, kbuf, vbuf) != -1);
-      assert(ndpi_serialize_string_uint32(&serializer, kbuf, i*i) != -1);
-      assert(ndpi_serialize_string_float(&serializer,  kbuf, (float)(i*i), "%f") != -1);
-      if ((i&0x3) == 0x3) ndpi_serialize_end_of_record(&serializer);
-    }
-
-    if (fmt == ndpi_serialization_format_json) {
-      assert(ndpi_serialize_start_of_list(&serializer, "List") != -1);
-
-      for(i=0; i<4; i++) {
-	char kbuf[32], vbuf[32];
-	snprintf(kbuf, sizeof(kbuf), "Ignored");
-	snprintf(vbuf, sizeof(vbuf), "Item %d", i);
-	assert(ndpi_serialize_uint32_uint32(&serializer, i, i*i) != -1);
-	assert(ndpi_serialize_string_string(&serializer, kbuf, vbuf) != -1);
-	assert(ndpi_serialize_string_float(&serializer,  kbuf, (float)(i*i), "%f") != -1);
-      }
-      assert(ndpi_serialize_end_of_list(&serializer) != -1);
-      assert(ndpi_serialize_string_string(&serializer, "Last", "Ok") != -1);
-
-      if(trace) {
-	u_int32_t buffer_len = 0;
-	char *buffer = ndpi_serializer_get_buffer(&serializer, &buffer_len);
-	printf("%s\n", buffer);
-      }
-    } else if (fmt == ndpi_serialization_format_csv) {
-      if(trace) {
-	u_int32_t buffer_len = 0;
-	char *buffer;
-
-	buffer = ndpi_serializer_get_header(&serializer, &buffer_len);
-	printf("%s\n", buffer);
-
-	buffer = ndpi_serializer_get_buffer(&serializer, &buffer_len);
-	printf("%s\n", buffer);
-      }
-
-    } else {
-      if(trace)
-	printf("Serialization size: %u\n", ndpi_serializer_get_buffer_len(&serializer));
-
-      assert(ndpi_init_deserializer(&deserializer, &serializer) != -1);
-
-      while(1) {
-	ndpi_serialization_type kt, et;
-
-	et = ndpi_deserialize_get_item_type(&deserializer, &kt);
-
-	if(et == ndpi_serialization_unknown) {
-	  break;
-        } else if(et == ndpi_serialization_end_of_record) {
-          if (trace) printf("EOR\n");
-	} else {
-	  u_int32_t k32, v32;
-	  ndpi_string ks, vs;
-	  float vf;
-
-	  switch(kt) {
-          case ndpi_serialization_uint32:
-            ndpi_deserialize_key_uint32(&deserializer, &k32);
-	    if(trace) printf("%u=", k32);
-	    break;
-          case ndpi_serialization_string:
-            ndpi_deserialize_key_string(&deserializer, &ks);
-            if (trace) {
-              u_int8_t bkp = ks.str[ks.str_len];
-	      ks.str[ks.str_len] = '\0';
-              printf("%s=", ks.str);
-	      ks.str[ks.str_len] = bkp;
-            }
-	    break;
-          default:
-            printf("ERROR: Unsupported TLV key type %u\n", kt);
-	    exit(0);
-	    return;
-	  }
-
-	  switch(et) {
-          case ndpi_serialization_uint32:
-	    assert(ndpi_deserialize_value_uint32(&deserializer, &v32) != -1);
-	    if(trace) printf("%u\n", v32);
-	    break;
-
-          case ndpi_serialization_string:
-	    assert(ndpi_deserialize_value_string(&deserializer, &vs) != -1);
-	    if(trace) {
-	      u_int8_t bkp = vs.str[vs.str_len];
-	      vs.str[vs.str_len] = '\0';
-	      printf("%s\n", vs.str);
-	      vs.str[vs.str_len] = bkp;
-	    }
-	    break;
-
-          case ndpi_serialization_float:
-	    assert(ndpi_deserialize_value_float(&deserializer, &vf) != -1);
-	    if(trace) printf("%f\n", vf);
-	    break;
-
-          default:
-	    if (trace) printf("\n");
-            printf("serializerUnitTest: unsupported type %u detected!\n", et);
-	    return;
-	  }
-	}
-
-	ndpi_deserialize_next(&deserializer);
-      }
-    }
-
-    ndpi_term_serializer(&serializer);
-  }
-
-  if (trace)
-    exit(0);
-}
-
-/* *********************************************** */
-
 // #define RUN_DATA_ANALYSIS_THEN_QUIT 1
 
 void analyzeUnitTest() {
@@ -3676,7 +3527,6 @@ int orginal_main(int argc, char **argv) {
     hllUnitTest();
     bitmapUnitTest();
     automataUnitTest();
-    serializerUnitTest();
     analyzeUnitTest();
     ndpi_self_check_host_match();
     analysisUnitTest();
diff --git a/tests/do.sh b/tests/do.sh
index 89a27c43a..3598146a5 100755
--- a/tests/do.sh
+++ b/tests/do.sh
@@ -3,6 +3,7 @@
 cd "$(dirname "${0}")"
 
 READER="../example/ndpiReader -p ../example/protos.txt -c ../example/categories.txt"
+UNIT="./unit/unit"
 
 RC=0
 PCAPS=`cd pcap; /bin/ls *.pcap`
@@ -45,8 +46,17 @@ check_results() {
 	done
 }
 
+check_unit() {
+	$UNIT
+	UNIT_RC=$?
+	if [ $UNIT_RC -ne 0 ]; then
+		RC=1
+	fi
+}
+
 fuzzy_testing
 build_results
 check_results
+check_unit
 
 exit $RC
diff --git a/tests/unit/Makefile.in b/tests/unit/Makefile.in
new file mode 100644
index 000000000..4acb001c8
--- /dev/null
+++ b/tests/unit/Makefile.in
@@ -0,0 +1,33 @@
+CC=@CC@
+CXX=@CXX@
+
+SRCHOME=../../src
+
+JSON_INC = $(shell pkg-config --cflags json-c)
+JSON_LIB = $(shell pkg-config --libs json-c)
+
+CFLAGS=-g -fPIC -DPIC -I$(SRCHOME)/include $(JSON_INC) @CFLAGS@
+LIBNDPI=$(SRCHOME)/lib/libndpi.a
+LDFLAGS=$(LIBNDPI) @PCAP_LIB@ @LIBS@ @ADDITIONAL_LIBS@ $(JSON_LIB) -lpthread -lm @LDFLAGS@
+HEADERS=$(SRCHOME)/include/ndpi_api.h $(SRCHOME)/include/ndpi_typedefs.h $(SRCHOME)/include/ndpi_protocol_ids.h
+OBJS=unit
+PREFIX?=@prefix@
+
+all: unit
+
+EXECUTABLE_SOURCES := unit.c
+COMMON_SOURCES := $(filter-out $(EXECUTABLE_SOURCES),$(wildcard *.c ))
+
+unit: $(LIBNDPI) unit.o
+	$(CC) $(CFLAGS) unit.o -o $@ $(LDFLAGS)
+
+%.o: %.c $(HEADERS) Makefile
+	$(CC) $(CFLAGS) -c $< -o $@
+
+clean:
+	/bin/rm -f *.o unit
+	/bin/rm -f .*.o.cmd .*.o.d
+	/bin/rm -rf build
+
+distclean: clean
+	/bin/rm -f Makefile
diff --git a/tests/unit/unit.c b/tests/unit/unit.c
new file mode 100644
index 000000000..c7fb292de
--- /dev/null
+++ b/tests/unit/unit.c
@@ -0,0 +1,245 @@
+/*
+ * unit.c
+ *
+ * Copyright (C) 2019-20 - ntop.org
+ *
+ * nDPI is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * nDPI is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with nDPI.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#ifdef linux
+#ifndef _GNU_SOURCE
+#define _GNU_SOURCE
+#endif
+#include <sched.h>
+#endif /* linux */
+
+#ifdef WIN32
+#include <winsock2.h>
+#include <process.h>
+#include <io.h>
+#define getopt getopt____
+#else
+#include <unistd.h>
+#include <netinet/in.h>
+#endif
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <getopt.h>
+#include <string.h>
+#include <stdarg.h>
+#include <search.h>
+#include <pcap.h>
+#include <signal.h>
+#include <pthread.h>
+#include <sys/socket.h>
+#include <assert.h>
+#include <math.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <sys/mman.h>
+#include <libgen.h>
+
+#include "json.h"
+
+#include "ndpi_config.h"
+#include "ndpi_api.h"
+
+static struct ndpi_detection_module_struct *ndpi_info_mod = NULL;
+static int verbose = 0;
+
+/* *********************************************** */
+
+int serializerUnitTest() {
+  ndpi_serializer serializer, deserializer;
+  int i, loop_id;
+  u_int8_t verbose = 0;
+  ndpi_serialization_format fmt;
+  u_int32_t buffer_len;
+  char *buffer;
+  enum json_tokener_error jerr;
+  json_object *j;
+
+  for(loop_id=0; loop_id<3; loop_id++) {
+    switch(loop_id) {
+    case 0:
+      if (verbose) printf("--- TLV test ---\n");
+      fmt = ndpi_serialization_format_tlv;
+      break;
+
+    case 1:
+      if (verbose) printf("--- JSON test ---\n");
+      fmt = ndpi_serialization_format_json;
+      break;
+
+    case 2:
+      if (verbose) printf("--- CSV test ---\n");
+      fmt = ndpi_serialization_format_csv;
+      break;
+    }
+    assert(ndpi_init_serializer(&serializer, fmt) != -1);
+
+    for(i=0; i<16; i++) {
+      char kbuf[32], vbuf[32];
+      snprintf(kbuf, sizeof(kbuf), "Key %d", i);
+      snprintf(vbuf, sizeof(vbuf), "Value %d", i);
+      assert(ndpi_serialize_uint32_uint32(&serializer, i, i*i) != -1);
+      assert(ndpi_serialize_uint32_string(&serializer, i, "Data") != -1);
+      assert(ndpi_serialize_string_string(&serializer, kbuf, vbuf) != -1);
+      assert(ndpi_serialize_string_uint32(&serializer, kbuf, i*i) != -1);
+      assert(ndpi_serialize_string_float(&serializer,  kbuf, (float)(i*i), "%f") != -1);
+      if ((i&0x3) == 0x3) ndpi_serialize_end_of_record(&serializer);
+    }
+
+    if (fmt == ndpi_serialization_format_json) {
+      assert(ndpi_serialize_start_of_list(&serializer, "List") != -1);
+
+      for(i=0; i<4; i++) {
+	char kbuf[32], vbuf[32];
+	snprintf(kbuf, sizeof(kbuf), "Ignored");
+	snprintf(vbuf, sizeof(vbuf), "Item %d", i);
+	assert(ndpi_serialize_uint32_uint32(&serializer, i, i*i) != -1);
+	assert(ndpi_serialize_string_string(&serializer, kbuf, vbuf) != -1);
+	assert(ndpi_serialize_string_float(&serializer,  kbuf, (float)(i*i), "%f") != -1);
+      }
+      assert(ndpi_serialize_end_of_list(&serializer) != -1);
+      assert(ndpi_serialize_string_string(&serializer, "Last", "Ok") != -1);
+
+      buffer = ndpi_serializer_get_buffer(&serializer, &buffer_len);
+
+      if(verbose)
+	printf("%s\n", buffer);
+
+      /* Decoding JSON to validate syntax */
+      jerr = json_tokener_success;
+      j = json_tokener_parse_verbose(buffer, &jerr);
+      if (j == NULL) {
+        printf("%s: ERROR (json validation failed)\n", __FUNCTION__);
+        return -1;
+      } else {
+        /* Validation ok */
+        json_object_put(j);
+      }
+
+    } else if (fmt == ndpi_serialization_format_csv) {
+      if(verbose) {
+	u_int32_t buffer_len = 0;
+	char *buffer;
+
+	buffer = ndpi_serializer_get_header(&serializer, &buffer_len);
+	printf("%s\n", buffer);
+
+	buffer = ndpi_serializer_get_buffer(&serializer, &buffer_len);
+	printf("%s\n", buffer);
+      }
+
+    } else {
+      if(verbose)
+	printf("Serialization size: %u\n", ndpi_serializer_get_buffer_len(&serializer));
+
+      assert(ndpi_init_deserializer(&deserializer, &serializer) != -1);
+
+      while(1) {
+	ndpi_serialization_type kt, et;
+
+	et = ndpi_deserialize_get_item_type(&deserializer, &kt);
+
+	if(et == ndpi_serialization_unknown) {
+	  break;
+        } else if(et == ndpi_serialization_end_of_record) {
+          if (verbose) printf("EOR\n");
+	} else {
+	  u_int32_t k32, v32;
+	  ndpi_string ks, vs;
+	  float vf;
+
+	  switch(kt) {
+          case ndpi_serialization_uint32:
+            ndpi_deserialize_key_uint32(&deserializer, &k32);
+	    if(verbose) printf("%u=", k32);
+	    break;
+          case ndpi_serialization_string:
+            ndpi_deserialize_key_string(&deserializer, &ks);
+            if (verbose) {
+              u_int8_t bkp = ks.str[ks.str_len];
+	      ks.str[ks.str_len] = '\0';
+              printf("%s=", ks.str);
+	      ks.str[ks.str_len] = bkp;
+            }
+	    break;
+          default:
+            printf("%s: ERROR (unsupported TLV key type %u)\n", __FUNCTION__, kt);
+	    return -1;
+	  }
+
+	  switch(et) {
+          case ndpi_serialization_uint32:
+	    assert(ndpi_deserialize_value_uint32(&deserializer, &v32) != -1);
+	    if(verbose) printf("%u\n", v32);
+	    break;
+
+          case ndpi_serialization_string:
+	    assert(ndpi_deserialize_value_string(&deserializer, &vs) != -1);
+	    if(verbose) {
+	      u_int8_t bkp = vs.str[vs.str_len];
+	      vs.str[vs.str_len] = '\0';
+	      printf("%s\n", vs.str);
+	      vs.str[vs.str_len] = bkp;
+	    }
+	    break;
+
+          case ndpi_serialization_float:
+	    assert(ndpi_deserialize_value_float(&deserializer, &vf) != -1);
+	    if(verbose) printf("%f\n", vf);
+	    break;
+
+          default:
+	    if (verbose) printf("\n");
+            printf("%s: ERROR (unsupported type %u detected)\n", __FUNCTION__, et);
+	    return -1;
+	  }
+	}
+
+	ndpi_deserialize_next(&deserializer);
+      }
+    }
+
+    ndpi_term_serializer(&serializer);
+  }
+
+  printf("%s: OK\n", __FUNCTION__);
+  return 0;
+}
+
+/* *********************************************** */
+
+int main(int argc, char **argv) {
+
+  if (ndpi_get_api_version() != NDPI_API_VERSION) {
+    printf("nDPI Library version mismatch: please make sure this code and the nDPI library are in sync\n");
+    return -1;
+  }
+
+  ndpi_info_mod = ndpi_init_detection_module(ndpi_no_prefs);
+
+  if (ndpi_info_mod == NULL)
+    return -1;
+
+  /* Tests */
+  if (serializerUnitTest() != 0) return -1;
+
+  return 0;
+}
+