fuzz: improvements (#2400)

Create the zip file with all the traces only once.

Add a new fuzzer to test "shoco" compression algorithm

9 files changed, 62 insertions, 34 deletions

diff --git a/.gitignore b/.gitignore
index 170c2923c..4605ff12b 100644
--- a/.gitignore
+++ b/.gitignore
@@ -61,6 +61,7 @@
 /fuzz/fuzz_alg_crc32_md5
 /fuzz/fuzz_alg_ses_des
 /fuzz/fuzz_alg_bytestream
+/fuzz/fuzz_alg_shoco
 /fuzz/fuzz_config
 /fuzz/fuzz_community_id
 /fuzz/fuzz_serialization
@@ -124,6 +125,7 @@
 /fuzz/fuzz_readerutils_workflow_seed_corpus.zip
 /fuzz/fuzz_readerutils_parseprotolist_seed_corpus.zip
 /fuzz/fuzz_*.dict
+/fuzz/testpcaps.zip
 /influxdb/Makefile
 /install-sh
 /libndpi.pc
diff --git a/fuzz/Makefile.am b/fuzz/Makefile.am
index b49a069d3..f59e8f76d 100644
--- a/fuzz/Makefile.am
+++ b/fuzz/Makefile.am
@@ -1,6 +1,6 @@
 bin_PROGRAMS = fuzz_process_packet fuzz_ndpi_reader fuzz_ndpi_reader_alloc_fail fuzz_ndpi_reader_payload_analyzer fuzz_quic_get_crypto_data fuzz_config fuzz_community_id fuzz_serialization fuzz_tls_certificate fuzz_dga fuzz_is_stun_udp fuzz_is_stun_tcp
 #Alghoritms
-bin_PROGRAMS += fuzz_alg_bins fuzz_alg_hll fuzz_alg_hw_rsi_outliers_da fuzz_alg_jitter fuzz_alg_ses_des fuzz_alg_crc32_md5 fuzz_alg_bytestream
+bin_PROGRAMS += fuzz_alg_bins fuzz_alg_hll fuzz_alg_hw_rsi_outliers_da fuzz_alg_jitter fuzz_alg_ses_des fuzz_alg_crc32_md5 fuzz_alg_bytestream fuzz_alg_shoco
 #Data structures
 bin_PROGRAMS += fuzz_ds_patricia fuzz_ds_ahocorasick fuzz_ds_libcache fuzz_ds_tree fuzz_ds_ptree fuzz_ds_hash fuzz_ds_cmsketch fuzz_ds_bitmap64_fuse fuzz_ds_domain_classify
 #Third party
@@ -208,6 +208,19 @@ fuzz_alg_bytestream_LINK=$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \
     $(LIBTOOLFLAGS) --mode=link $(CXX) @NDPI_CFLAGS@ $(AM_CXXFLAGS) $(CXXFLAGS) \
     $(fuzz_alg_bytestream_LDFLAGS) @NDPI_LDFLAGS@ $(LDFLAGS) -o $@
 
+fuzz_alg_shoco_SOURCES = fuzz_alg_shoco.cpp
+fuzz_alg_shoco_CXXFLAGS = @NDPI_CFLAGS@ $(CXXFLAGS) -I ../src/lib/third_party/include/
+fuzz_alg_shoco_LDADD = ../src/lib/libndpi.a $(ADDITIONAL_LIBS)
+fuzz_alg_shoco_LDFLAGS = $(LIBS)
+if HAS_FUZZLDFLAGS
+fuzz_alg_shoco_CXXFLAGS += $(LIB_FUZZING_ENGINE)
+fuzz_alg_shoco_LDFLAGS += $(LIB_FUZZING_ENGINE)
+endif
+# force usage of CXX for linker
+fuzz_alg_shoco_LINK=$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \
+    $(LIBTOOLFLAGS) --mode=link $(CXX) @NDPI_CFLAGS@ $(AM_CXXFLAGS) $(CXXFLAGS) \
+    $(fuzz_alg_shoco_LDFLAGS) @NDPI_LDFLAGS@ $(LDFLAGS) -o $@
+
 fuzz_alg_ses_des_SOURCES = fuzz_alg_ses_des.cpp fuzz_common_code.c
 fuzz_alg_ses_des_CXXFLAGS = @NDPI_CFLAGS@ $(CXXFLAGS)
 fuzz_alg_ses_des_CFLAGS = @NDPI_CFLAGS@ $(CXXFLAGS)
@@ -624,14 +637,17 @@ fuzz_readerutils_parseprotolist_LINK=$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOL
 # see https://github.com/google/oss-fuzz/tree/master/projects/ndpi
 testpcaps := $(wildcard ../tests/cfgs/default/pcap/*.pcap*)
 
-fuzz_ndpi_reader_seed_corpus.zip: $(testpcaps)
-	zip -j fuzz_ndpi_reader_seed_corpus.zip $(testpcaps)
+testpcaps.zip: $(testpcaps)
+	zip -j testpcaps.zip $(testpcaps)
+
+fuzz_ndpi_reader_seed_corpus.zip: testpcaps.zip
+	cp testpcaps.zip fuzz_ndpi_reader_seed_corpus.zip
 
-fuzz_ndpi_reader_alloc_fail_seed_corpus.zip: $(testpcaps)
-	zip -j fuzz_ndpi_reader_alloc_fail_seed_corpus.zip $(testpcaps)
+fuzz_ndpi_reader_alloc_fail_seed_corpus.zip: testpcaps.zip
+	cp testpcaps.zip fuzz_ndpi_reader_alloc_fail_seed_corpus.zip
 
-fuzz_ndpi_reader_payload_analyzer_seed_corpus.zip: $(testpcaps)
-	zip -j fuzz_ndpi_reader_payload_analyzer_seed_corpus.zip $(testpcaps)
+fuzz_ndpi_reader_payload_analyzer_seed_corpus.zip: testpcaps.zip
+	cp testpcaps.zip fuzz_ndpi_reader_payload_analyzer_seed_corpus.zip
 
 files_corpus_fuzz_quic_get_crypto_data :=  $(wildcard corpus/fuzz_quic_get_crypto_data/*)
 fuzz_quic_get_crypto_data_seed_corpus.zip: $(files_corpus_fuzz_quic_get_crypto_data)
diff --git a/fuzz/fuzz_alg_shoco.cpp b/fuzz/fuzz_alg_shoco.cpp
new file mode 100644
index 000000000..cf8d31bb3
--- /dev/null
+++ b/fuzz/fuzz_alg_shoco.cpp
@@ -0,0 +1,21 @@
+#include <stdint.h>
+#include "shoco.h"
+#include "fuzzer/FuzzedDataProvider.h"
+
+extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
+  FuzzedDataProvider fuzzed_data(data, size);
+  const char *in;
+  size_t in_len, out_len;
+  char out[4096], orig[4096];
+
+  /* No memory allocations involved */
+
+  std::string s = fuzzed_data.ConsumeRemainingBytesAsString().c_str();
+  in = s.c_str();
+  in_len = strlen(in);
+
+  out_len = shoco_compress(in, in_len, out, sizeof(out));
+  shoco_decompress(out, out_len, orig, sizeof(orig));
+
+  return 0;
+}
diff --git a/fuzz/fuzz_ds_domain_classify.cpp b/fuzz/fuzz_ds_domain_classify.cpp
index 8428c2b52..8e19b4725 100644
--- a/fuzz/fuzz_ds_domain_classify.cpp
+++ b/fuzz/fuzz_ds_domain_classify.cpp
@@ -6,6 +6,8 @@
 #include <assert.h>
 #include "fuzzer/FuzzedDataProvider.h"
 
+static struct ndpi_detection_module_struct *ndpi_struct = NULL;
+
 extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
   FuzzedDataProvider fuzzed_data(data, size);
   u_int16_t i, num_iteration, is_added = 0;
@@ -13,13 +15,11 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
   ndpi_domain_classify *d;
   u_int16_t class_id;
   std::string value, value_added;
-  struct ndpi_detection_module_struct *ndpi_struct;
-  NDPI_PROTOCOL_BITMASK all;
-  
-  ndpi_struct = ndpi_init_detection_module(NULL);
-  NDPI_BITMASK_SET_ALL(all);
-  ndpi_set_protocol_detection_bitmask2(ndpi_struct, &all);
-  
+
+  if (ndpi_struct == NULL) {
+    fuzz_init_detection_module(&ndpi_struct, NULL);
+  }
+
   /* To allow memory allocation failures */
   fuzz_set_alloc_callbacks_and_seed(size);
 
@@ -58,6 +58,5 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
   ndpi_domain_classify_size(d);
   ndpi_domain_classify_free(d);
   
-  ndpi_exit_detection_module(ndpi_struct);
   return 0;
 }
diff --git a/fuzz/fuzz_ds_hash.cpp b/fuzz/fuzz_ds_hash.cpp
index 9bb065365..07aa732a6 100644
--- a/fuzz/fuzz_ds_hash.cpp
+++ b/fuzz/fuzz_ds_hash.cpp
@@ -8,7 +8,7 @@
 
 extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
   FuzzedDataProvider fuzzed_data(data, size);
-  u_int16_t i, rc, num_iteration, data_len, is_added = 0;
+  u_int16_t value16, i, rc, num_iteration, data_len, is_added = 0;
   std::vector<char>value_added;
   ndpi_str_hash *h = NULL;
 
@@ -30,7 +30,7 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
     data_len = fuzzed_data.ConsumeIntegralInRange(0, 127);
     std::vector<char>data = fuzzed_data.ConsumeBytes<char>(data_len);
 
-    rc = ndpi_hash_add_entry(&h, data.data(), data.size(), (u_int16_t)i);
+    rc = ndpi_hash_add_entry(&h, data.data(), data.size(), i);
     /* Keep one random entry really added */
     if (rc == 0 && fuzzed_data.ConsumeBool()) {
       value_added = data;
@@ -41,8 +41,6 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
   /* "Random" search */
   num_iteration = fuzzed_data.ConsumeIntegral<u_int8_t>();
   for (i = 0; i < num_iteration; i++) {
-    u_int16_t value16;
-    
     data_len = fuzzed_data.ConsumeIntegralInRange(0, 127);
     std::vector<char>data = fuzzed_data.ConsumeBytes<char>(data_len);
 
@@ -50,7 +48,6 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
   }
   /* Search of an added entry */
   if (is_added) {
-    u_int16_t value16;
     ndpi_hash_find_entry(h, value_added.data(), value_added.size(), &value16);
   }
 
diff --git a/src/lib/ndpi_community_id.c b/src/lib/ndpi_community_id.c
index de66b2954..6074599f1 100644
--- a/src/lib/ndpi_community_id.c
+++ b/src/lib/ndpi_community_id.c
@@ -67,10 +67,7 @@
 /* **************************************************** */
 
 static u_int16_t ndpi_community_id_buf_copy(u_int8_t * const dst, const void * const src, u_int16_t len) {
-  if(src)
-    memcpy(dst, src, len);
-  else
-    memset(dst, 0, len);
+  memcpy(dst, src, len);
 
   return len;
 }
diff --git a/src/lib/ndpi_domain_classify.c b/src/lib/ndpi_domain_classify.c
index f62800527..068bb9faf 100644
--- a/src/lib/ndpi_domain_classify.c
+++ b/src/lib/ndpi_domain_classify.c
@@ -143,15 +143,6 @@ u_int32_t ndpi_domain_classify_add_domains(struct ndpi_detection_module_struct *
 
 /* ********************************************************** */
 
-bool ndpi_domain_classify_finalize(ndpi_domain_classify *s) {
-  if(!s)
-    return(false);
-
-  return(true);
-}
-
-/* ********************************************************** */
-
 bool ndpi_domain_classify_hostname(struct ndpi_detection_module_struct *ndpi_mod,
 				   ndpi_domain_classify *s,
 				   u_int16_t *class_id /* out */,
diff --git a/src/lib/ndpi_main.c b/src/lib/ndpi_main.c
index ada595971..5056cf99e 100644
--- a/src/lib/ndpi_main.c
+++ b/src/lib/ndpi_main.c
@@ -7981,7 +7981,6 @@ int ndpi_enable_loaded_categories(struct ndpi_detection_module_struct *ndpi_str)
   }
 #else
   ndpi_domain_classify_free(ndpi_str->custom_categories.sc_hostnames);
-  // ndpi_domain_classify_finalize(ndpi_str->custom_categories.sc_hostnames_shadow);
   ndpi_str->custom_categories.sc_hostnames        = ndpi_str->custom_categories.sc_hostnames_shadow;
   ndpi_str->custom_categories.sc_hostnames_shadow = ndpi_domain_classify_alloc();
 #endif
diff --git a/src/lib/ndpi_utils.c b/src/lib/ndpi_utils.c
index 49d86e28d..f75eb3d9c 100644
--- a/src/lib/ndpi_utils.c
+++ b/src/lib/ndpi_utils.c
@@ -2302,6 +2302,9 @@ int ndpi_hash_find_entry(ndpi_str_hash *h, char *key, u_int key_len, u_int16_t *
   ndpi_str_hash_priv *h_priv = (ndpi_str_hash_priv *)h;
   ndpi_str_hash_priv *item;
 
+  if(!key || key_len == 0)
+    return(2);
+
   HASH_FIND(hh, h_priv, key, key_len, item);
 
   if (item != NULL) {
@@ -2319,6 +2322,9 @@ int ndpi_hash_add_entry(ndpi_str_hash **h, char *key, u_int8_t key_len, u_int16_
   ndpi_str_hash_priv *h_priv = (ndpi_str_hash_priv *)*h;
   ndpi_str_hash_priv *item;
 
+  if(!key || key_len == 0)
+    return(3);
+
   HASH_FIND(hh, h_priv, key, key_len, item);
 
   if(item != NULL) {