Provide some API functions for convenience. (#1456)

* Extended JSON serializsation: risk, risk score, confidence

Signed-off-by: Toni Uhlig <matzeton@googlemail.com>

5 files changed, 225 insertions, 9 deletions

diff --git a/src/include/ndpi_api.h.in b/src/include/ndpi_api.h.in
index a60bcb6ee..abd67655f 100644
--- a/src/include/ndpi_api.h.in
+++ b/src/include/ndpi_api.h.in
@@ -332,6 +332,40 @@ extern "C" {
 					 struct ndpi_flow_struct *flow);
 
   /**
+   * Get the app protocol of the passed flows for the detected module
+   *
+   *
+   * @par    ndpi_struct  = the detection module
+   * @par    flow         = the flow given for the detection module
+   * @return the ID of the app protocol detected
+   *
+   */
+  u_int16_t ndpi_get_flow_appprotocol(struct ndpi_detection_module_struct *ndpi_str, struct ndpi_flow_struct *flow);
+
+  /**
+   * Get the category of the passed flows for the detected module
+   *
+   *
+   * @par    ndpi_struct  = the detection module
+   * @par    flow         = the flow given for the detection module
+   * @return the ID of the category
+   *
+   */
+  ndpi_protocol_category_t ndpi_get_flow_category(struct ndpi_detection_module_struct *ndpi_str, struct ndpi_flow_struct *flow);
+
+  /**
+   * Get the ndpi protocol data of the passed flows for the detected module
+   *
+   *
+   * @par    ndpi_struct  = the detection module
+   * @par    flow         = the flow given for the detection module
+   * @par    ndpi_proto   = the output struct where to store the requested information
+   *
+   */
+  void ndpi_get_flow_ndpi_proto(struct ndpi_detection_module_struct *ndpi_str, struct ndpi_flow_struct *flow,
+                    struct ndpi_proto * ndpi_proto);
+
+  /**
    * API call that is called internally by ndpi_detection_process_packet or by apps
    * that want to avoid calling ndpi_detection_process_packet as they have already
    * parsed the packet and thus want to avoid this.
@@ -1402,6 +1436,14 @@ extern "C" {
   int ndpi_serialize_start_of_block(ndpi_serializer *serializer, const char *key);
 
   /**
+   * Serialize the start of a block with a numeric key
+   * @param serializer The serializer handle
+   * @param key The numeric key as 32-bit unsigned integer.
+   * @return 0 on success, a negative number otherwise
+   */
+  int ndpi_serialize_start_of_block_uint32(ndpi_serializer *serializer, u_int32_t key);
+
+  /**
    * Serialize the end of a block
    * @param serializer The serializer handle
    * @param key The field name or ID
@@ -1563,8 +1605,12 @@ extern "C" {
   u_int8_t ndpi_is_protocol_detected(struct ndpi_detection_module_struct *ndpi_str,
 				     ndpi_protocol proto);
   void ndpi_serialize_risk(ndpi_serializer *serializer, ndpi_risk risk);
+  void ndpi_serialize_risk_score(ndpi_serializer *serializer, ndpi_risk_enum risk);
+  void ndpi_serialize_confidence(ndpi_serializer *serializer, ndpi_confidence_t confidence);
   void ndpi_serialize_proto(struct ndpi_detection_module_struct *ndpi_struct,
-                            ndpi_serializer *serializer, ndpi_risk_enum risk,
+                            ndpi_serializer *serializer,
+                            ndpi_risk_enum risk,
+                            ndpi_confidence_t confidence,
                             ndpi_protocol l7_protocol);
   const char* ndpi_risk2str(ndpi_risk_enum risk);
   const char* ndpi_severity2str(ndpi_risk_severity s);
diff --git a/src/lib/ndpi_main.c b/src/lib/ndpi_main.c
index 37d73172c..c0a871cc5 100644
--- a/src/lib/ndpi_main.c
+++ b/src/lib/ndpi_main.c
@@ -6490,6 +6490,27 @@ u_int16_t ndpi_get_flow_masterprotocol(struct ndpi_detection_module_struct *ndpi
 
 /* ********************************************************************************* */
 
+u_int16_t ndpi_get_flow_appprotocol(struct ndpi_detection_module_struct *ndpi_str, struct ndpi_flow_struct *flow) {
+  return(flow->detected_protocol_stack[0]);
+}
+
+/* ********************************************************************************* */
+
+ndpi_protocol_category_t ndpi_get_flow_category(struct ndpi_detection_module_struct *ndpi_str, struct ndpi_flow_struct *flow)
+{
+  return(flow->category);
+}
+
+void ndpi_get_flow_ndpi_proto(struct ndpi_detection_module_struct *ndpi_str, struct ndpi_flow_struct *flow,
+                struct ndpi_proto * ndpi_proto)
+{
+  ndpi_proto->master_protocol = ndpi_get_flow_masterprotocol(ndpi_str, flow);
+  ndpi_proto->app_protocol = ndpi_get_flow_appprotocol(ndpi_str, flow);
+  ndpi_proto->category = ndpi_get_flow_category(ndpi_str, flow);
+}
+
+/* ********************************************************************************* */
+
 static void ndpi_int_change_flow_protocol(struct ndpi_detection_module_struct *ndpi_str, struct ndpi_flow_struct *flow,
 					  u_int16_t upper_detected_protocol, u_int16_t lower_detected_protocol,
 					  ndpi_confidence_t confidence) {
diff --git a/src/lib/ndpi_serializer.c b/src/lib/ndpi_serializer.c
index 1699644a4..76c764405 100644
--- a/src/lib/ndpi_serializer.c
+++ b/src/lib/ndpi_serializer.c
@@ -1983,6 +1983,21 @@ int ndpi_serialize_start_of_block(ndpi_serializer *_serializer,
 
 /* ********************************** */
 
+/* Serialize start of nested block with a numeric key */
+int ndpi_serialize_start_of_block_uint32(ndpi_serializer *_serializer, u_int32_t key) {
+  char buf[11];
+  int written = snprintf(buf, sizeof(buf), "%u", key);
+
+  if (written <= 0 || written == sizeof(buf))
+  {
+    return(-1);
+  }
+
+  return(ndpi_serialize_start_of_block_binary(_serializer, buf, written));
+}
+
+/* ********************************** */
+
 /* Serialize end of nested block (JSON only)*/
 int ndpi_serialize_end_of_block(ndpi_serializer *_serializer) {
   ndpi_private_serializer *serializer = (ndpi_private_serializer*)_serializer;
diff --git a/src/lib/ndpi_utils.c b/src/lib/ndpi_utils.c
index 29cb94695..8cbc2e2df 100644
--- a/src/lib/ndpi_utils.c
+++ b/src/lib/ndpi_utils.c
@@ -1109,7 +1109,7 @@ void ndpi_serialize_risk(ndpi_serializer *serializer,
                          ndpi_risk risk) {
   u_int32_t i;
 
-  if (risk == NDPI_NO_RISK) {
+  if (risk == 0) {
     return;
   }
 
@@ -1117,24 +1117,68 @@ void ndpi_serialize_risk(ndpi_serializer *serializer,
   for(i = 0; i < NDPI_MAX_RISK; i++) {
     ndpi_risk_enum r = (ndpi_risk_enum)i;
 
-    if(NDPI_ISSET_BIT(risk, r))
-      ndpi_serialize_uint32_string(serializer, i, ndpi_risk2str(r));
+    if(NDPI_ISSET_BIT(risk, r)) {
+      ndpi_risk_info const * const risk_info = ndpi_risk2severity(r);
+      if (risk_info == NULL)
+        continue;
+
+      ndpi_serialize_start_of_block_uint32(serializer, i);
+      ndpi_serialize_string_string(serializer, "risk", ndpi_risk2str(risk_info->risk));
+      ndpi_serialize_string_string(serializer, "severity", ndpi_severity2str(risk_info->severity));
+      ndpi_serialize_risk_score(serializer, r);
+      ndpi_serialize_end_of_block(serializer);
+    }
   }
 
   ndpi_serialize_end_of_block(serializer);
 }
 
- /* ********************************** */
+/* ********************************** */
+
+void ndpi_serialize_risk_score(ndpi_serializer *serializer,
+                               ndpi_risk_enum risk)
+{
+  u_int16_t rs, rs_client = 0, rs_server = 0;
+
+  if (risk == NDPI_NO_RISK) {
+    return;
+  }
+
+  ndpi_serialize_start_of_block(serializer, "risk_score");
+  rs = ndpi_risk2score(risk, &rs_client, &rs_server);
+  ndpi_serialize_string_uint32(serializer, "total", rs);
+  ndpi_serialize_string_uint32(serializer, "client", rs_client);
+  ndpi_serialize_string_uint32(serializer, "server", rs_server);
+  ndpi_serialize_end_of_block(serializer);
+}
+
+/* ********************************** */
+
+void ndpi_serialize_confidence(ndpi_serializer *serializer,
+                               ndpi_confidence_t confidence)
+{
+  if (confidence == NDPI_CONFIDENCE_UNKNOWN) {
+    return;
+  }
+
+  ndpi_serialize_start_of_block(serializer, "confidence");
+  ndpi_serialize_uint32_string(serializer, (u_int32_t)confidence, ndpi_confidence_get_name(confidence));
+  ndpi_serialize_end_of_block(serializer);
+}
+
+/* ********************************** */
 
 void ndpi_serialize_proto(struct ndpi_detection_module_struct *ndpi_struct,
                           ndpi_serializer *serializer,
                           ndpi_risk_enum risk,
+                          ndpi_confidence_t confidence,
                           ndpi_protocol l7_protocol)
 {
   char buf[64];
 
   ndpi_serialize_start_of_block(serializer, "ndpi");
   ndpi_serialize_risk(serializer, risk);
+  ndpi_serialize_confidence(serializer, confidence);
   ndpi_serialize_string_string(serializer, "proto", ndpi_protocol2name(ndpi_struct, l7_protocol, buf, sizeof(buf)));
   ndpi_protocol_breed_t breed =
     ndpi_get_proto_breed(ndpi_struct,
@@ -1156,7 +1200,7 @@ int ndpi_dpi2json(struct ndpi_detection_module_struct *ndpi_struct,
 
   if(flow == NULL) return(-1);
 
-  ndpi_serialize_proto(ndpi_struct, serializer, flow->risk, l7_protocol);
+  ndpi_serialize_proto(ndpi_struct, serializer, flow->risk, flow->confidence, l7_protocol);
 
   switch(l7_protocol.master_protocol ? l7_protocol.master_protocol : l7_protocol.app_protocol) {
   case NDPI_PROTOCOL_IP_ICMP:
diff --git a/tests/unit/unit.c b/tests/unit/unit.c
index 44d637e2c..69d44f3c2 100644
--- a/tests/unit/unit.c
+++ b/tests/unit/unit.c
@@ -54,6 +54,7 @@
 
 #include "ndpi_config.h"
 #include "ndpi_api.h"
+#include "ndpi_define.h"
 
 #ifdef HAVE_LIBJSON_C
 #include "json.h" /* JSON-C */
@@ -138,12 +139,12 @@ int serializerUnitTest() {
 
     } else if (fmt == ndpi_serialization_format_csv) {
       if(verbose) {
-	u_int32_t buffer_len = 0;
-	char *buffer;
 
+	buffer_len = 0;
 	buffer = ndpi_serializer_get_header(&serializer, &buffer_len);
 	printf("%s\n", buffer);
 
+	buffer_len = 0;
 	buffer = ndpi_serializer_get_buffer(&serializer, &buffer_len);
 	printf("%s\n", buffer);
       }
@@ -228,13 +229,101 @@ int serializerUnitTest() {
     ndpi_term_serializer(&serializer);
   }
 
-  printf("%s                      OK\n", __FUNCTION__);
+  printf("%30s                      OK\n", __FUNCTION__);
 #endif
   return 0;
 }
 
 /* *********************************************** */
 
+int serializeProtoUnitTest(void)
+{
+#ifdef HAVE_LIBJSON_C
+  ndpi_serializer serializer;
+  int loop_id;
+  ndpi_serialization_format fmt = {0};
+  u_int32_t buffer_len;
+  char * buffer;
+
+  for(loop_id=0; loop_id<3; loop_id++) {
+    switch(loop_id) {
+    case 0:
+      if (verbose) printf("--- TLV test ---\n");
+      fmt = ndpi_serialization_format_tlv;
+      break;
+
+    case 1:
+      if (verbose) printf("--- JSON test ---\n");
+      fmt = ndpi_serialization_format_json;
+      break;
+
+    case 2:
+      if (verbose) printf("--- CSV test ---\n");
+      fmt = ndpi_serialization_format_csv;
+      break;
+    }
+    assert(ndpi_init_serializer(&serializer, fmt) != -1);
+
+    ndpi_protocol ndpi_proto = { .master_protocol = NDPI_PROTOCOL_TLS,
+                                 .app_protocol = NDPI_PROTOCOL_FACEBOOK,
+                                 .category = NDPI_PROTOCOL_CATEGORY_SOCIAL_NETWORK };
+    ndpi_risk risks = 0;
+    NDPI_SET_BIT(risks, NDPI_MALFORMED_PACKET);
+    NDPI_SET_BIT(risks, NDPI_TLS_WEAK_CIPHER);
+    NDPI_SET_BIT(risks, NDPI_TLS_OBSOLETE_VERSION);
+    NDPI_SET_BIT(risks, NDPI_TLS_SELFSIGNED_CERTIFICATE);
+    ndpi_serialize_proto(ndpi_info_mod, &serializer, risks, NDPI_CONFIDENCE_DPI, ndpi_proto);
+
+    if (fmt == ndpi_serialization_format_json)
+    {
+      buffer_len = 0;
+      buffer = ndpi_serializer_get_buffer(&serializer, &buffer_len);
+      char const * const expected_json_str = "{\"ndpi\": {\"flow_risk\": {\"6\": {\"risk\":\"Self-signed Certificate\",\"severity\":\"High\",\"risk_score\": {\"total\":500,\"client\":450,\"server\":50}},\"7\": {\"risk\":\"Obsolete TLS Version (1.1 or older)\",\"severity\":\"High\",\"risk_score\": {\"total\":510,\"client\":455,\"server\":55}},\"8\": {\"risk\":\"Weak TLS Cipher\",\"severity\":\"High\",\"risk_score\": {\"total\":250,\"client\":225,\"server\":25}},\"17\": {\"risk\":\"Malformed Packet\",\"severity\":\"Low\",\"risk_score\": {\"total\":260,\"client\":130,\"server\":130}}},\"confidence\": {\"4\":\"DPI\"},\"proto\":\"TLS.Facebook\",\"breed\":\"Fun\",\"category\":\"SocialNetwork\"}}";
+
+      if (strncmp(buffer, expected_json_str, buffer_len) != 0)
+      {
+        printf("%s: ERROR: expected JSON str: \"%s\"\n", __FUNCTION__, expected_json_str);
+        printf("%s: ERROR: got JSON str: \"%.*s\"\n", __FUNCTION__, (int)buffer_len, buffer);
+        return -1;
+      }
+
+      if(verbose)
+        printf("%s\n", buffer);
+
+      /* Decoding JSON to validate syntax */
+      enum json_tokener_error jerr = json_tokener_success;
+      json_object * const j = json_tokener_parse_verbose(buffer, &jerr);
+      if (j == NULL) {
+        printf("%s: ERROR (json validation failed)\n", __FUNCTION__);
+        return -1;
+      } else {
+        /* Validation ok */
+        json_object_put(j);
+      }
+    } else if (fmt == ndpi_serialization_format_csv)
+    {
+      if(verbose) {
+        buffer_len = 0;
+        buffer = ndpi_serializer_get_header(&serializer, &buffer_len);
+        printf("%s\n", buffer);
+
+        buffer_len = 0;
+        buffer = ndpi_serializer_get_buffer(&serializer, &buffer_len);
+        printf("%s\n", buffer);
+      }
+    }
+
+    ndpi_term_serializer(&serializer);
+  }
+
+  printf("%30s                      OK\n", __FUNCTION__);
+#endif
+
+  return 0;
+}
+
+/* *********************************************** */
+
 int main(int argc, char **argv) {
   int c;
   
@@ -262,6 +351,7 @@ int main(int argc, char **argv) {
     
   /* Tests */
   if (serializerUnitTest() != 0) return -1;
+  if (serializeProtoUnitTest() != 0) return -1;
 
   return 0;
 }