Improved nDPI JSON serialization. (#1689)

* fixed autoconf CFLAGS/LDFLAGS MSAN issue which could lead to build errors
 * introduced portable version of gmtime_r aka ndpi_gmtime_r
 * do as most as possible of the serialization work in ndpi_utils.c
 * use flow2json in ndpiReader

Signed-off-by: lns <matzeton@googlemail.com>

3 files changed, 42 insertions, 201 deletions

diff --git a/example/ndpiReader.c b/example/ndpiReader.c
index c06be6733..b197ec8e6 100644
--- a/example/ndpiReader.c
+++ b/example/ndpiReader.c
@@ -1060,12 +1060,15 @@ static void parseOptions(int argc, char **argv) {
     }
   }
 
-  if ((serialization_fp == NULL && serialization_format != ndpi_serialization_format_unknown) ||
-      (serialization_fp != NULL && serialization_format == ndpi_serialization_format_unknown))
+  if (serialization_fp == NULL && serialization_format != ndpi_serialization_format_unknown)
   {
-    printf("Serializing detection results to a file requires command line arguments -k AND -K\n");
+    printf("Serializing detection results to a file requires command line arguments `-k'\n");
     exit(1);
   }
+  if (serialization_fp != NULL && serialization_format == ndpi_serialization_format_unknown)
+  {
+    serialization_format = ndpi_serialization_format_json;
+  }
 
   if(extcap_exit)
     exit(0);
@@ -1584,8 +1587,10 @@ static void printFlow(u_int32_t id, struct ndpi_flow_info *flow, u_int16_t threa
     if(flow->ssh_tls.tls_subjectDN) fprintf(out, "[Subject: %s]", flow->ssh_tls.tls_subjectDN);
 
     if(flow->ssh_tls.encrypted_sni.esni) {
+      char unknown_cipher[8];
       fprintf(out, "[ESNI: %s]", flow->ssh_tls.encrypted_sni.esni);
-      fprintf(out, "[ESNI Cipher: %s]", ndpi_cipher2str(flow->ssh_tls.encrypted_sni.cipher_suite));
+      fprintf(out, "[ESNI Cipher: %s]",
+              ndpi_cipher2str(flow->ssh_tls.encrypted_sni.cipher_suite, unknown_cipher));
     }
 
     if(flow->ssh_tls.sha1_cert_fingerprint_set) {
@@ -1605,8 +1610,8 @@ static void printFlow(u_int32_t id, struct ndpi_flow_info *flow, u_int16_t threa
     if(flow->ssh_tls.notBefore && flow->ssh_tls.notAfter) {
       char notBefore[32], notAfter[32];
       struct tm a, b;
-      struct tm *before = gmtime_r(&flow->ssh_tls.notBefore, &a);
-      struct tm *after  = gmtime_r(&flow->ssh_tls.notAfter, &b);
+      struct tm *before = ndpi_gmtime_r(&flow->ssh_tls.notBefore, &a);
+      struct tm *after  = ndpi_gmtime_r(&flow->ssh_tls.notAfter, &b);
 
       strftime(notBefore, sizeof(notBefore), "%Y-%m-%d %H:%M:%S", before);
       strftime(notAfter, sizeof(notAfter), "%Y-%m-%d %H:%M:%S", after);
@@ -1614,7 +1619,11 @@ static void printFlow(u_int32_t id, struct ndpi_flow_info *flow, u_int16_t threa
       fprintf(out, "[Validity: %s - %s]", notBefore, notAfter);
     }
 
-    if(flow->ssh_tls.server_cipher != '\0') fprintf(out, "[Cipher: %s]", ndpi_cipher2str(flow->ssh_tls.server_cipher));
+    char unknown_cipher[8];
+    if(flow->ssh_tls.server_cipher != '\0')
+    {
+      fprintf(out, "[Cipher: %s]", ndpi_cipher2str(flow->ssh_tls.server_cipher, unknown_cipher));
+    }
     if(flow->bittorent_hash != NULL) fprintf(out, "[BT Hash: %s]", flow->bittorent_hash);
     if(flow->dhcp_fingerprint != NULL) fprintf(out, "[DHCP Fingerprint: %s]", flow->dhcp_fingerprint);
     if(flow->dhcp_class_ident) fprintf(out, "[DHCP Class Ident: %s]",
@@ -1639,69 +1648,17 @@ static void printFlowSerialized(u_int16_t thread_id,
 {
   char *json_str = NULL;
   u_int32_t json_str_len = 0;
-  ndpi_serializer * const serializer = &ndpi_thread_info[thread_id].workflow->ndpi_serializer;
+  ndpi_serializer * const serializer = &flow->ndpi_flow_serializer;
   //float data_ratio = ndpi_data_ratio(flow->src2dst_bytes, flow->dst2src_bytes);
   double f = (double)flow->first_seen_ms, l = (double)flow->last_seen_ms;
-  u_int8_t known_tls;
-  char buf[64];
   float data_ratio = ndpi_data_ratio(flow->src2dst_bytes, flow->dst2src_bytes);
 
-  ndpi_reset_serializer(serializer);
-
   ndpi_serialize_string_uint32(serializer, "flow_id", flow->flow_id);
-  ndpi_serialize_string_uint32(serializer, "l4_protocol", flow->protocol);
   ndpi_serialize_string_float(serializer, "first_seen_ms", f, "%.3f");
   ndpi_serialize_string_float(serializer, "last_seen_ms", l, "%.3f");
   ndpi_serialize_string_float(serializer, "duration_ms", (l-f)/1000.0, "%.3f");
-  ndpi_serialize_string_string(serializer, "src_name", flow->src_name);
-  ndpi_serialize_string_string(serializer, "dst_name", flow->dst_name);
-  ndpi_serialize_string_uint32(serializer, "src_port", ntohs(flow->src_port));
-  ndpi_serialize_string_uint32(serializer, "dst_port", ntohs(flow->dst_port));
-  ndpi_serialize_string_uint32(serializer, "ip_version", flow->ip_version);
   ndpi_serialize_string_uint32(serializer, "vlan_id", flow->vlan_id);
   ndpi_serialize_string_uint32(serializer, "bidirectional", flow->bidirectional);
-  ndpi_serialize_string_uint32(serializer, "encrypted",
-                               ndpi_is_encrypted_proto(ndpi_thread_info[thread_id].workflow->ndpi_struct,
-                                                       flow->detected_protocol));
-  ndpi_serialize_string_string(serializer, "confidence",
-                               ndpi_confidence_get_name(flow->confidence));
-  ndpi_serialize_string_uint32(serializer, "category_id",
-                               flow->detected_protocol.category);
-  ndpi_serialize_string_string(
-    serializer, "category_name",
-    ndpi_category_get_name(
-      ndpi_thread_info[thread_id].workflow->ndpi_struct,
-      flow->detected_protocol.category
-    )
-  );
-
-  ndpi_serialize_string_string(serializer, "l7_protocol_id",
-                               ndpi_protocol2id(ndpi_thread_info[thread_id].workflow->ndpi_struct,
-                               flow->detected_protocol, buf, sizeof(buf)));
-  ndpi_serialize_string_string(serializer, "l7_protocol_name",
-                               ndpi_protocol2name(ndpi_thread_info[thread_id].workflow->ndpi_struct,
-                               flow->detected_protocol, buf, sizeof(buf)));
-  ndpi_serialize_start_of_list(serializer, "risks");
-  if (flow->risk != NDPI_NO_RISK)
-  {
-    u_int32_t i;
-
-    for(i = 0; i < NDPI_MAX_RISK; ++i)
-    {
-      if (NDPI_ISSET_BIT(flow->risk, i) != 0)
-      {
-        ndpi_serialize_string_string(serializer, "str", ndpi_risk2str(i));
-      }
-    }
-  }
-  ndpi_serialize_end_of_list(serializer);
-  {
-    u_int16_t cli_score, srv_score;
-    ndpi_serialize_string_uint32(serializer, "risks_score",
-                                 ndpi_risk2score(flow->risk, &cli_score, &srv_score));
-  }
-  ndpi_serialize_string_string(serializer, "host_server_name",
-                               flow->host_server_name);
 
   /* XFER Packets/Bytes */
   ndpi_serialize_start_of_block(serializer, "xfer");
@@ -1802,137 +1759,6 @@ static void printFlowSerialized(u_int16_t thread_id,
   ndpi_serialize_string_uint32(serializer, "c_to_s_init_win", flow->c_to_s_init_win);
   ndpi_serialize_string_uint32(serializer, "s_to_c_init_win", flow->s_to_c_init_win);
 
-  /* Protocol specific serialization */
-  ndpi_serialize_start_of_block(serializer, "l7_protocol_data");
-  if (flow->ssh_tls.server_info[0] != '\0')
-  {
-    ndpi_serialize_string_string(serializer, "server_info", flow->ssh_tls.server_info);
-  }
-
-  if (flow->ssh_tls.server_names != NULL)
-  {
-    ndpi_serialize_string_string(serializer, "server_names", flow->ssh_tls.server_names);
-  }
-
-  if (flow->ssh_tls.ssl_version != 0)
-  {
-    ndpi_ssl_version2str(buf, sizeof(buf), flow->ssh_tls.ssl_version, &known_tls);
-    ndpi_serialize_string_string(serializer, "version", buf);
-  }
-
-  if (flow->ssh_tls.ja3_client[0] != '\0')
-  {
-    ndpi_serialize_string_string(serializer, "ja3_client", flow->ssh_tls.ja3_client);
-  }
-
-  if (flow->ssh_tls.ja3_server[0] != '\0')
-  {
-    ndpi_serialize_string_string(serializer, "ja3_server", flow->ssh_tls.ja3_server);
-  }
-
-  if (flow->ssh_tls.tls_issuerDN != NULL)
-  {
-    ndpi_serialize_string_string(serializer, "issuerDN", flow->ssh_tls.tls_issuerDN);
-  }
-
-  if (flow->ssh_tls.tls_subjectDN != NULL)
-  {
-    ndpi_serialize_string_string(serializer, "subjectDN", flow->ssh_tls.tls_subjectDN);
-  }
-
-  if (flow->ssh_tls.client_hassh[0] != '\0')
-  {
-    ndpi_serialize_string_string(serializer, "client_hassh", flow->ssh_tls.client_hassh);
-  }
-
-  if (flow->ssh_tls.server_hassh[0] != '\0')
-  {
-    ndpi_serialize_string_string(serializer, "server_hassh", flow->ssh_tls.server_hassh);
-  }
-
-  if (flow->http.user_agent[0] != '\0')
-  {
-    ndpi_serialize_string_string(serializer, "user_agent", flow->http.user_agent);
-  }
-
-  if (flow->http.url[0] != '\0')
-  {
-    ndpi_risk_enum risk = ndpi_validate_url(flow->http.url);
-    if (risk != NDPI_NO_RISK)
-    {
-      NDPI_SET_BIT(flow->risk, risk);
-    }
-
-    ndpi_serialize_string_string(serializer, "url", flow->http.url);
-    ndpi_serialize_string_uint32(serializer, "code", flow->http.response_status_code);
-    if (flow->http.request_content_type[0] != '\0')
-    {
-      ndpi_serialize_string_string(serializer, "req_content_type",
-                                   flow->http.request_content_type);
-    }
-
-    if (flow->http.content_type[0] != '\0')
-    {
-      ndpi_serialize_string_string(serializer, "content_type",
-                                   flow->http.content_type);
-    }
-  }
-
-  switch (flow->info_type)
-  {
-    case INFO_INVALID:
-      break;
-
-    case INFO_GENERIC:
-      if (flow->info[0] != '\0')
-      {
-        ndpi_serialize_string_string(serializer, "info", flow->info);
-      }
-      break;
-
-    case INFO_KERBEROS:
-      if (flow->kerberos.domain[0] != '\0' ||
-          flow->kerberos.hostname[0] != '\0' ||
-          flow->kerberos.username[0] != '\0')
-      {
-        ndpi_serialize_string_string(serializer, "domain",
-                                     flow->kerberos.domain);
-        ndpi_serialize_string_string(serializer, "hostname",
-                                     flow->kerberos.hostname);
-        ndpi_serialize_string_string(serializer, "username",
-                                     flow->kerberos.username);
-      }
-      break;
-
-    case INFO_SOFTETHER:
-      ndpi_serialize_string_string(serializer, "client_ip", flow->softether.ip);
-      ndpi_serialize_string_string(serializer, "client_port", flow->softether.port);
-      ndpi_serialize_string_string(serializer, "hostname", flow->softether.hostname);
-      ndpi_serialize_string_string(serializer, "fqdn", flow->softether.fqdn);
-      break;
-
-    case INFO_FTP_IMAP_POP_SMTP:
-      ndpi_serialize_string_string(serializer, "username",
-                                   flow->ftp_imap_pop_smtp.username);
-      ndpi_serialize_string_string(serializer, "password",
-                                   flow->ftp_imap_pop_smtp.password);
-      ndpi_serialize_string_uint32(serializer, "auth_failed",
-                                   flow->ftp_imap_pop_smtp.auth_failed);
-      break;
-
-    case INFO_TLS_QUIC_ALPN_VERSION:
-      ndpi_serialize_string_string(serializer, "alpn", flow->tls_quic.alpn);
-      ndpi_serialize_string_string(serializer, "supported_versions",
-                                   flow->tls_quic.tls_supported_versions);
-      break;
-
-    case INFO_TLS_QUIC_ALPN_ONLY:
-      ndpi_serialize_string_string(serializer, "alpn", flow->tls_quic.alpn);
-      break;
-  }
-
-  ndpi_serialize_end_of_block(serializer);
-
   json_str = ndpi_serializer_get_buffer(serializer, &json_str_len);
   if (json_str == NULL || json_str_len == 0)
   {
diff --git a/example/reader_util.c b/example/reader_util.c
index 48f522cbf..7ccaba1fc 100644
--- a/example/reader_util.c
+++ b/example/reader_util.c
@@ -434,13 +434,7 @@ struct ndpi_workflow* ndpi_workflow_init(const struct ndpi_workflow_prefs * pref
   if(do_init_flows_root)
     workflow->ndpi_flows_root = ndpi_calloc(workflow->prefs.num_roots, sizeof(void *));
 
-  if (serialization_format != ndpi_serialization_format_unknown &&
-      ndpi_init_serializer(&workflow->ndpi_serializer,
-                           serialization_format) != 0)
-  {
-    LOG(NDPI_LOG_ERROR, "serializer initialization failed\n");
-    exit(-1);
-  }
+  workflow->ndpi_serialization_format = serialization_format;
 
   return workflow;
 }
@@ -532,6 +526,7 @@ static void ndpi_free_flow_data_analysis(struct ndpi_flow_info *flow) {
 void ndpi_flow_info_free_data(struct ndpi_flow_info *flow) {
 
   ndpi_free_flow_info_half(flow);
+  ndpi_term_serializer(&flow->ndpi_flow_serializer);
   ndpi_free_flow_data_analysis(flow);
   ndpi_free_flow_tls_data(flow);
 
@@ -550,8 +545,6 @@ void ndpi_flow_info_free_data(struct ndpi_flow_info *flow) {
 void ndpi_workflow_free(struct ndpi_workflow * workflow) {
   u_int i;
 
-  ndpi_term_serializer(&workflow->ndpi_serializer);
-
   for(i=0; i<workflow->prefs.num_roots; i++)
     ndpi_tdestroy(workflow->ndpi_flows_root[i], ndpi_flow_info_freer);
 
@@ -1282,6 +1275,26 @@ void process_ndpi_collected_info(struct ndpi_workflow * workflow, struct ndpi_fl
 	workflow->__flow_detected_callback(workflow, flow, workflow->__flow_detected_udata);
     }
 
+    if (workflow->ndpi_serialization_format != ndpi_serialization_format_unknown)
+    {
+      if (ndpi_init_serializer(&flow->ndpi_flow_serializer,
+                               workflow->ndpi_serialization_format) != 0)
+      {
+        LOG(NDPI_LOG_ERROR, "ndpi serializer init failed\n");
+        exit(-1);
+      }
+      if (ndpi_flow2json(workflow->ndpi_struct, flow->ndpi_flow,
+                         flow->ip_version, flow->protocol,
+                         flow->src_ip, flow->dst_ip,
+                         &flow->src_ip6, &flow->dst_ip6,
+                         flow->src_port, flow->dst_port,
+                         flow->detected_protocol,
+                         &flow->ndpi_flow_serializer) != 0)
+      {
+        LOG(NDPI_LOG_ERROR, "flow2json failed\n");
+        exit(-1);
+      }
+    }
     ndpi_free_flow_info_half(flow);
   }
 }
diff --git a/example/reader_util.h b/example/reader_util.h
index e61167837..dab9ae57d 100644
--- a/example/reader_util.h
+++ b/example/reader_util.h
@@ -236,6 +236,8 @@ typedef struct ndpi_flow_info {
     } softether;
   };
 
+  ndpi_serializer ndpi_flow_serializer;
+
   char flow_extra_info[16];
   char host_server_name[80]; /* Hostname/SNI */
   char *bittorent_hash;
@@ -348,7 +350,7 @@ typedef struct ndpi_workflow {
   u_int32_t num_allocated_flows;
 
   /* CSV,TLV,JSON serialization interface */
-  ndpi_serializer ndpi_serializer;
+  ndpi_serialization_format ndpi_serialization_format;
 } ndpi_workflow_t;