Added ndpi_serialize_raw_record() API call

3 files changed, 101 insertions, 54 deletions

diff --git a/src/include/ndpi_api.h.in b/src/include/ndpi_api.h.in
index 375e65910..06e68a49d 100644
--- a/src/include/ndpi_api.h.in
+++ b/src/include/ndpi_api.h.in
@@ -945,7 +945,8 @@ extern "C" {
 				  const char *format /* e.f. "%.2f" */);
   int ndpi_serialize_string_boolean(ndpi_serializer *serializer,
 				    const char *key, u_int8_t value);
-
+  int ndpi_serialize_raw_record(ndpi_serializer *_serializer,
+				u_char *record, u_int32_t record_len);
   int ndpi_serialize_end_of_record(ndpi_serializer *serializer);
   int ndpi_serialize_start_of_block(ndpi_serializer *serializer,
 				    const char *key);
diff --git a/src/lib/ndpi_analyze.c b/src/lib/ndpi_analyze.c
index d1335bb64..f648e92ae 100644
--- a/src/lib/ndpi_analyze.c
+++ b/src/lib/ndpi_analyze.c
@@ -40,7 +40,7 @@ void ndpi_init_data_analysis(struct ndpi_analyze_struct *ret, u_int16_t _max_ser
   u_int32_t len;
 
   memset(ret, 0, sizeof(struct ndpi_analyze_struct));
-  
+
   if(_max_series_len > MAX_SERIES_LEN) _max_series_len = MAX_SERIES_LEN;
   ret->num_values_array_len = _max_series_len;
 
@@ -61,8 +61,8 @@ struct ndpi_analyze_struct* ndpi_alloc_data_analysis(u_int16_t _max_series_len)
   struct ndpi_analyze_struct *ret = ndpi_malloc(sizeof(struct ndpi_analyze_struct));
 
   if(ret != NULL)
-    ndpi_init_data_analysis(ret, _max_series_len);  
-  
+    ndpi_init_data_analysis(ret, _max_series_len);
+
   return(ret);
 }
 
@@ -89,18 +89,18 @@ void ndpi_data_add_value(struct ndpi_analyze_struct *s, const u_int32_t value) {
   }
 
   s->sum_total += value, s->num_data_entries++;
-  
+
   if(s->num_values_array_len) {
     s->values[s->next_value_insert_index] = value;
 
     if(++s->next_value_insert_index == s->num_values_array_len)
       s->next_value_insert_index = 0;
   }
-  
+
   /* Update stddev */
   tmp_mu = s->stddev.mu;
   s->stddev.mu = ((s->stddev.mu * (s->num_data_entries - 1)) + value) / s->num_data_entries;
-  s->stddev.q = s->stddev.q + (value - tmp_mu)*(value - s->stddev.mu);    
+  s->stddev.q = s->stddev.q + (value - tmp_mu)*(value - s->stddev.mu);
 }
 
 /* ********************************************************************************* */
@@ -137,10 +137,10 @@ float ndpi_data_window_average(struct ndpi_analyze_struct *s) {
   if(s->num_values_array_len) {
     float   sum = 0.0;
     u_int16_t i, n = ndpi_min(s->num_data_entries, s->num_values_array_len);
-    
+
     for(i=0; i<n; i++)
       sum += s->values[i];
-    
+
     return((float)sum / (float)n);
   } else
     return(0);
@@ -155,17 +155,17 @@ float ndpi_data_entropy(struct ndpi_analyze_struct *s) {
   if(s->num_values_array_len) {
     int i;
     float sum = 0.0, total = 0.0;
-    
+
     for(i=0; i<s->num_values_array_len; i++)
       total += s->values[i];
-    
+
     for (i=0; i<s->num_values_array_len; i++) {
       float tmp = (float)s->values[i] / (float)total;
-      
+
       if(tmp > FLT_EPSILON)
-	sum -= tmp * logf(tmp);    
+	sum -= tmp * logf(tmp);
     }
-    
+
     return(sum / logf(2.0));
   } else
     return(0);
@@ -176,10 +176,10 @@ float ndpi_data_entropy(struct ndpi_analyze_struct *s) {
 void ndpi_data_print_window_values(struct ndpi_analyze_struct *s) {
   if(s->num_values_array_len) {
     u_int16_t i, n = ndpi_min(s->num_data_entries, s->num_values_array_len);
-    
+
     for(i=0; i<n; i++)
       printf("[%u: %u]", i, s->values[i]);
-    
+
     printf("\n");
   }
 }
@@ -196,7 +196,7 @@ void ndpi_data_print_window_values(struct ndpi_analyze_struct *s) {
 float ndpi_data_ratio(u_int32_t sent, u_int32_t rcvd) {
   float s = (float)((int64_t)sent +  (int64_t)rcvd);
   float d = (float)((int64_t)sent -  (int64_t)rcvd);
-  
+
   return((s == 0) ? 0 : (d/s));
 }
 
diff --git a/src/lib/ndpi_serializer.c b/src/lib/ndpi_serializer.c
index 57c616044..521e841d0 100644
--- a/src/lib/ndpi_serializer.c
+++ b/src/lib/ndpi_serializer.c
@@ -3,9 +3,6 @@
  *
  * Copyright (C) 2011-20 - ntop.org
  *
- * This file is part of nDPI, an open source deep packet inspection
- * library based on the OpenDPI and PACE technology by ipoque GmbH
- *
  * nDPI is free software: you can redistribute it and/or modify
  * it under the terms of the GNU Lesser General Public License as published by
  * the Free Software Foundation, either version 3 of the License, or
@@ -170,7 +167,7 @@ int ndpi_init_serializer_ll(ndpi_serializer *_serializer,
   memset(serializer, 0, sizeof(ndpi_private_serializer));
 
   serializer->initial_buffer_size = serializer->buffer_size = buffer_size;
-  serializer->buffer      = (u_int8_t *) malloc(serializer->buffer_size * sizeof(u_int8_t));
+  serializer->buffer      = (u_int8_t *) calloc(serializer->buffer_size, sizeof(u_int8_t));
 
   if(serializer->buffer == NULL)
     return(-1);
@@ -197,6 +194,36 @@ int ndpi_init_serializer(ndpi_serializer *_serializer,
 
 /* ********************************** */
 
+static inline int ndpi_extend_serializer_buffer(ndpi_serializer *_serializer, u_int32_t min_len) {
+  u_int32_t new_size;
+  void *r;
+  ndpi_private_serializer *serializer = (ndpi_private_serializer*)_serializer;
+
+  if(min_len < NDPI_SERIALIZER_DEFAULT_BUFFER_INCR) {
+    if(serializer->initial_buffer_size < NDPI_SERIALIZER_DEFAULT_BUFFER_INCR) {
+      if(min_len < serializer->initial_buffer_size)
+        min_len = serializer->initial_buffer_size;
+    } else {
+      min_len = NDPI_SERIALIZER_DEFAULT_BUFFER_INCR;
+    }
+  }
+
+  new_size = serializer->buffer_size + min_len;
+  new_size = ((new_size / 4) + 1) * 4; /* required by zmq encryption */
+
+  r = realloc((void *) serializer->buffer, new_size);
+
+  if(r == NULL)
+    return(-1);
+
+  serializer->buffer = r;
+  serializer->buffer_size = new_size;
+
+  return(0);
+}
+
+/* ********************************** */
+
 char* ndpi_serializer_get_buffer(ndpi_serializer *_serializer, u_int32_t *buffer_len) {
   ndpi_private_serializer *serializer = (ndpi_private_serializer*)_serializer;
   char *buf = (char*)serializer->buffer;
@@ -265,36 +292,6 @@ void ndpi_term_serializer(ndpi_serializer *_serializer) {
 
 /* ********************************** */
 
-static inline int ndpi_extend_serializer_buffer(ndpi_serializer *_serializer, u_int32_t min_len) {
-  u_int32_t new_size;
-  void *r;
-  ndpi_private_serializer *serializer = (ndpi_private_serializer*)_serializer;
-
-  if(min_len < NDPI_SERIALIZER_DEFAULT_BUFFER_INCR) {
-    if(serializer->initial_buffer_size < NDPI_SERIALIZER_DEFAULT_BUFFER_INCR) {
-      if(min_len < serializer->initial_buffer_size)
-        min_len = serializer->initial_buffer_size;
-    } else {
-      min_len = NDPI_SERIALIZER_DEFAULT_BUFFER_INCR;
-    }
-  }
-
-  new_size = serializer->buffer_size + min_len;
-  new_size = ((new_size / 4) + 1) * 4; /* required by zmq encryption */
-
-  r = realloc((void *) serializer->buffer, new_size);
-
-  if(r == NULL)
-    return(-1);
-
-  serializer->buffer = r;
-  serializer->buffer_size = new_size;
-
-  return(0);
-}
-
-/* ********************************** */
-
 static inline void ndpi_serialize_single_uint8(ndpi_private_serializer *serializer,
 					       u_int8_t s) {
   u_int8_t v = s;
@@ -431,11 +428,60 @@ static inline void ndpi_deserialize_single_string(ndpi_private_deserializer *des
 
 /* ********************************** */
 
+/*
+  This function helps extending the existing serializer by adding a new
+  element in the array. This element is handled as raw without any
+  further check whatsoever
+*/
+int ndpi_serialize_raw_record(ndpi_serializer *_serializer,
+			      u_char *record, u_int32_t record_len) {
+  ndpi_private_serializer *serializer = (ndpi_private_serializer*)_serializer;
+  u_int32_t buff_diff = serializer->buffer_size - serializer->status.size_used;
+  u_int16_t needed = record_len;
+  u_int8_t add_comma = 0;
+    
+  if(serializer->fmt == ndpi_serialization_format_json) {
+    needed += 1;
+
+    if(serializer->status.size_used == 3) /* Empty buffer [{} */
+      serializer->status.size_used = 2; /* Remove {} */
+    else
+      needed += 2, add_comma = 1;
+  }
+  
+  if(buff_diff < needed) {
+    if(ndpi_extend_serializer_buffer(_serializer, needed - buff_diff) < 0)
+      return(-1);
+    
+    buff_diff = serializer->buffer_size - serializer->status.size_used;
+  }
+
+  if(serializer->fmt == ndpi_serialization_format_json) {   
+    if(add_comma)
+      serializer->buffer[serializer->status.size_used-1] = ',';
+    else
+      serializer->status.size_used--;
+  }
+  
+  memcpy(&serializer->buffer[serializer->status.size_used], record, record_len);
+  serializer->status.size_used += record_len;
+
+  if(serializer->fmt == ndpi_serialization_format_json) {
+    serializer->buffer[serializer->status.size_used] = ']';
+    if(add_comma) serializer->status.size_used++;
+  }
+  
+  ndpi_serialize_end_of_record(_serializer);
+  
+  return(0);
+}
+
+/* ********************************** */
+
 int ndpi_serialize_end_of_record(ndpi_serializer *_serializer) {
   ndpi_private_serializer *serializer = (ndpi_private_serializer*)_serializer;
   u_int32_t buff_diff = serializer->buffer_size - serializer->status.size_used;
-  u_int16_t needed =
-    sizeof(u_int8_t) /* type */;
+  u_int16_t needed = sizeof(u_int8_t) /* type */;
 
   if(serializer->fmt == ndpi_serialization_format_json)
     needed += 1;
@@ -452,7 +498,7 @@ int ndpi_serialize_end_of_record(ndpi_serializer *_serializer) {
       serializer->status.size_used += snprintf((char *) &serializer->buffer[serializer->status.size_used],
 					       buff_diff, "]");
     }
-    
+
     serializer->status.flags |= NDPI_SERIALIZER_STATUS_ARRAY | NDPI_SERIALIZER_STATUS_EOR;
     serializer->status.flags &= ~NDPI_SERIALIZER_STATUS_COMMA;
   } else {