Add support for multiline json

2 files changed, 78 insertions, 35 deletions

diff --git a/src/include/ndpi_typedefs.h b/src/include/ndpi_typedefs.h
index 9bcdfad87..e3a649892 100644
--- a/src/include/ndpi_typedefs.h
+++ b/src/include/ndpi_typedefs.h
@@ -1756,7 +1756,8 @@ typedef enum {
   ndpi_serialization_format_unknown = 0,
   ndpi_serialization_format_tlv,
   ndpi_serialization_format_json,
-  ndpi_serialization_format_csv
+  ndpi_serialization_format_csv,
+  ndpi_serialization_format_multiline_json
 } ndpi_serialization_format;
 
 /* Note:
@@ -1821,6 +1822,7 @@ typedef struct {
   ndpi_serialization_format fmt;
   char csv_separator[2];
   u_int8_t has_snapshot;
+  u_int8_t multiline_json_array;
   ndpi_private_serializer_status snapshot;
 } ndpi_private_serializer;
 
diff --git a/src/lib/ndpi_serializer.c b/src/lib/ndpi_serializer.c
index 08b7734f7..0758c7272 100644
--- a/src/lib/ndpi_serializer.c
+++ b/src/lib/ndpi_serializer.c
@@ -177,8 +177,12 @@ void ndpi_reset_serializer(ndpi_serializer *_serializer) {
     serializer->status.buffer.size_used = 0;
     buff_diff = serializer->buffer.size - serializer->status.buffer.size_used;
 
-    /* Note: please keep a space at the beginning as it is used for arrays when an end-of-record is used */
-    serializer->status.buffer.size_used += ndpi_snprintf((char *) &serializer->buffer.data[serializer->status.buffer.size_used], buff_diff, " {}");
+    if (serializer->multiline_json_array) {
+      serializer->status.buffer.size_used += ndpi_snprintf((char *) &serializer->buffer.data[serializer->status.buffer.size_used], buff_diff, "{}");
+    } else {
+      /* Note: please keep a space at the beginning as it is used for arrays when an end-of-record is used */
+      serializer->status.buffer.size_used += ndpi_snprintf((char *) &serializer->buffer.data[serializer->status.buffer.size_used], buff_diff, " {}");
+    }
   } else if(serializer->fmt == ndpi_serialization_format_csv) {
     serializer->status.header.size_used = 0;
     serializer->status.buffer.size_used = 0;
@@ -215,6 +219,11 @@ int ndpi_init_serializer_ll(ndpi_serializer *_serializer,
 
   memset(serializer, 0, sizeof(ndpi_private_serializer));
 
+  if (fmt == ndpi_serialization_format_multiline_json) {
+    fmt = ndpi_serialization_format_json;
+    serializer->multiline_json_array = 1;
+  }
+
   serializer->fmt = fmt;
 
   if (buffer_size < 3 ||
@@ -614,10 +623,18 @@ int ndpi_serialize_raw_record(ndpi_serializer *_serializer,
   if(serializer->fmt == ndpi_serialization_format_json) {
     needed += 1;
 
-    if(serializer->status.buffer.size_used == 3) /* Empty buffer [{} */
-      serializer->status.buffer.size_used = 2; /* Remove {} */
-    else
-      needed += 2, add_comma = 1;
+    if (serializer->multiline_json_array) {
+      if(serializer->status.buffer.size_used == 2) /* Empty buffer {} */
+        serializer->status.buffer.size_used = 0; /* Remove {} */
+      else
+        needed += 2;
+
+    } else {
+      if(serializer->status.buffer.size_used == 3) /* Empty buffer [{} */
+        serializer->status.buffer.size_used = 2; /* Remove {} */
+      else
+        needed += 2, add_comma = 1;
+    }
   }
 
   if(buff_diff < needed) {
@@ -628,18 +645,22 @@ int ndpi_serialize_raw_record(ndpi_serializer *_serializer,
   }
 
   if(serializer->fmt == ndpi_serialization_format_json) {
-    if(add_comma)
-      serializer->buffer.data[serializer->status.buffer.size_used-1] = ',';
-    else
-      serializer->status.buffer.size_used--;
+    if (!serializer->multiline_json_array) {
+      if (add_comma)
+        serializer->buffer.data[serializer->status.buffer.size_used-1] = ',';
+      else
+        serializer->status.buffer.size_used--;
+    }
   }
 
   memcpy(&serializer->buffer.data[serializer->status.buffer.size_used], record, record_len);
   serializer->status.buffer.size_used += record_len;
 
   if(serializer->fmt == ndpi_serialization_format_json) {
-    serializer->buffer.data[serializer->status.buffer.size_used] = ']';
-    if(add_comma) serializer->status.buffer.size_used++;
+    if (!serializer->multiline_json_array) {
+      serializer->buffer.data[serializer->status.buffer.size_used] = ']';
+      if(add_comma) serializer->status.buffer.size_used++;
+    }
   }
 
   ndpi_serialize_end_of_record(_serializer);
@@ -665,20 +686,26 @@ int ndpi_serialize_end_of_record(ndpi_serializer *_serializer) {
   }
 
   if(serializer->fmt == ndpi_serialization_format_csv) {
-    serializer->buffer.data[serializer->status.buffer.size_used] = '\n';
-    serializer->status.buffer.size_used += 1;
+    serializer->buffer.data[serializer->status.buffer.size_used++] = '\n';
     serializer->buffer.data[serializer->status.buffer.size_used] = '\0';
     serializer->status.flags |= NDPI_SERIALIZER_STATUS_HDR_DONE;
     serializer->status.flags |= NDPI_SERIALIZER_STATUS_EOR;
+
   } else if(serializer->fmt == ndpi_serialization_format_json) {
-    if(!(serializer->status.flags & NDPI_SERIALIZER_STATUS_ARRAY)) {
-      serializer->buffer.data[0] = '[';
-      serializer->status.buffer.size_used += ndpi_snprintf((char *) &serializer->buffer.data[serializer->status.buffer.size_used],
-					       buff_diff, "]");
-    }
 
+    if(serializer->multiline_json_array) {
+      serializer->buffer.data[serializer->status.buffer.size_used++] = '\n';
+      serializer->buffer.data[serializer->status.buffer.size_used] = '\0';
+
+    } else {
+      if(!(serializer->status.flags & NDPI_SERIALIZER_STATUS_ARRAY)) {
+        serializer->buffer.data[0] = '[';
+        serializer->status.buffer.size_used += ndpi_snprintf((char *) &serializer->buffer.data[serializer->status.buffer.size_used], buff_diff, "]");
+      }
+    }
     serializer->status.flags |= NDPI_SERIALIZER_STATUS_ARRAY | NDPI_SERIALIZER_STATUS_EOR;
     serializer->status.flags &= ~NDPI_SERIALIZER_STATUS_COMMA;
+
   } else /* ndpi_serialization_format_tlv */ {
     serializer->buffer.data[serializer->status.buffer.size_used++] = ndpi_serialization_end_of_record;
   }
@@ -709,18 +736,26 @@ static inline void ndpi_serialize_json_pre(ndpi_serializer *_serializer) {
   ndpi_private_serializer *serializer = (ndpi_private_serializer*)_serializer;
 
   if(serializer->status.flags & NDPI_SERIALIZER_STATUS_EOR) {
-    serializer->status.buffer.size_used--; /* Remove ']' */
     serializer->status.flags &= ~NDPI_SERIALIZER_STATUS_EOR;
-    serializer->buffer.data[serializer->status.buffer.size_used++] = ',';
+    if(serializer->multiline_json_array) {
+      serializer->buffer.data[serializer->status.buffer.size_used++] = '\n';
+    } else {
+      serializer->status.buffer.size_used--; /* Remove ']' */
+      serializer->buffer.data[serializer->status.buffer.size_used++] = ',';
+    }
     serializer->buffer.data[serializer->status.buffer.size_used++] = '{';
+
   } else {
-    if(serializer->status.flags & NDPI_SERIALIZER_STATUS_ARRAY)
-      serializer->status.buffer.size_used--; /* Remove ']' */
+    if(!serializer->multiline_json_array) {
+      if(serializer->status.flags & NDPI_SERIALIZER_STATUS_ARRAY)
+        serializer->status.buffer.size_used--; /* Remove ']' */
+    }
 
     serializer->status.buffer.size_used--; /* Remove '}' */
 
     if(serializer->status.flags & NDPI_SERIALIZER_STATUS_LIST) {
-      serializer->status.buffer.size_used--; /* Remove ']' */
+      if(!serializer->multiline_json_array)
+        serializer->status.buffer.size_used--; /* Remove ']' */
       if(serializer->status.flags & NDPI_SERIALIZER_STATUS_SOL)
         serializer->status.flags &= ~NDPI_SERIALIZER_STATUS_SOL;
       else
@@ -739,20 +774,24 @@ static inline void ndpi_serialize_json_pre(ndpi_serializer *_serializer) {
 static inline int ndpi_serialize_json_post(ndpi_serializer *_serializer) {
   ndpi_private_serializer *serializer = (ndpi_private_serializer*)_serializer;
 
-  if(serializer->status.flags & NDPI_SERIALIZER_STATUS_LIST) {
-    if(serializer->status.buffer.size_used >= serializer->buffer.size)
-      return -1;
-    serializer->buffer.data[serializer->status.buffer.size_used++] = ']';
+  if (!serializer->multiline_json_array) {
+    if(serializer->status.flags & NDPI_SERIALIZER_STATUS_LIST) {
+      if(serializer->status.buffer.size_used >= serializer->buffer.size)
+        return -1;
+      serializer->buffer.data[serializer->status.buffer.size_used++] = ']';
+    }
   }
 
   if(serializer->status.buffer.size_used >= serializer->buffer.size)
     return -1;
   serializer->buffer.data[serializer->status.buffer.size_used++] = '}';
 
-  if(serializer->status.flags & NDPI_SERIALIZER_STATUS_ARRAY) {
-    if(serializer->status.buffer.size_used >= serializer->buffer.size)
-      return -1;
-    serializer->buffer.data[serializer->status.buffer.size_used++] = ']';
+  if (!serializer->multiline_json_array) {
+    if(serializer->status.flags & NDPI_SERIALIZER_STATUS_ARRAY) {
+      if(serializer->status.buffer.size_used >= serializer->buffer.size)
+        return -1;
+      serializer->buffer.data[serializer->status.buffer.size_used++] = ']';
+    }
   }
 
   serializer->status.flags |= NDPI_SERIALIZER_STATUS_COMMA;
@@ -2391,9 +2430,11 @@ void ndpi_serializer_rollback_snapshot(ndpi_serializer *_serializer) {
 
     if(serializer->fmt == ndpi_serialization_format_json) {
       if(serializer->status.flags & NDPI_SERIALIZER_STATUS_ARRAY) {
-        serializer->buffer.data[serializer->status.buffer.size_used-1] = ']';
+        if (!serializer->multiline_json_array)
+          serializer->buffer.data[serializer->status.buffer.size_used-1] = ']';
       } else {
-        serializer->buffer.data[0] = ' ';
+        if (!serializer->multiline_json_array)
+          serializer->buffer.data[0] = ' ';
         serializer->buffer.data[serializer->status.buffer.size_used-1] = '}';
       }
     }