all fixed

2 files changed, 375 insertions, 4 deletions

diff --git a/example/ndpiReader.c b/example/ndpiReader.c
index 003737212..a4afef821 100644
--- a/example/ndpiReader.c
+++ b/example/ndpiReader.c
@@ -1421,7 +1421,7 @@ static void node_idle_scan_walker(const void *node, ndpi_VISIT which, int depth,
   }
 }
 
-/* *********************************************** */  
+/* *********************************************** */
 
 /**
  * @brief On Protocol Discover - demo callback
@@ -1609,7 +1609,7 @@ char* formatTraffic(float numBits, int bits, char *buf) {
   return(buf);
 }
 
-/* *********************************************** */  
+/* *********************************************** */
 
 /**
  * @brief Packets stats format
@@ -2176,6 +2176,337 @@ static void printResults(u_int64_t processing_time_usec, u_int64_t setup_time_us
         ndpi_twalk(ndpi_thread_info[thread_id].workflow->ndpi_flows_root[i], node_print_known_proto_walker, &thread_id);
     }
 
+   //
+
+      ndpi_host_ja3_fingerprints *ja3ByHostsHashT = NULL; //outer hash table
+      ndpi_ja3_fingerprints_host *hostByJA3C_ht = NULL; //for client
+      ndpi_ja3_fingerprints_host *hostByJA3S_ht = NULL; //for server
+
+      int i = 0;
+      for(i = 0; i < num_flows; i++){
+         ndpi_host_ja3_fingerprints *ja3ByHostFound = NULL;
+         ndpi_ja3_fingerprints_host *hostByJA3Found = NULL;
+
+         //check if this is a ssh-ssl flow
+         if(all_flows[i].flow->ssh_ssl.ja3_client[0] != '\0'){
+            //looking if the host is already in the hash table
+            HASH_FIND_INT(ja3ByHostsHashT, &(all_flows[i].flow->src_ip), ja3ByHostFound);
+
+            //host ip -> ja3
+            if(ja3ByHostFound == NULL){
+               //adding the new host
+               ndpi_host_ja3_fingerprints *newHost = malloc(sizeof(ndpi_host_ja3_fingerprints));
+               newHost->host_client_info_hasht = NULL;
+               newHost->host_server_info_hasht = NULL;
+               newHost->ip_string = all_flows[i].flow->src_name;
+               newHost->ip = all_flows[i].flow->src_ip;
+               newHost->dns_name = all_flows[i].flow->ssh_ssl.client_info;
+
+               ndpi_ja3_info *newJA3 = malloc(sizeof(ndpi_ja3_info));
+               newJA3->ja3 = all_flows[i].flow->ssh_ssl.ja3_client;
+               newJA3->unsafe_cipher = all_flows[i].flow->ssh_ssl.client_unsafe_cipher;
+               //adding the new ja3 fingerprint
+               HASH_ADD_KEYPTR(hh, newHost->host_client_info_hasht, newJA3->ja3, strlen(newJA3->ja3), newJA3);
+               //adding the new host
+               HASH_ADD_INT(ja3ByHostsHashT, ip, newHost);
+            }else{
+               //host already in the hash table
+               ndpi_ja3_info *infoFound = NULL;
+               HASH_FIND_STR(ja3ByHostFound->host_client_info_hasht, all_flows[i].flow->ssh_ssl.ja3_client, infoFound);
+               if(infoFound == NULL){
+                  ndpi_ja3_info *newJA3 = malloc(sizeof(ndpi_ja3_info));
+                  newJA3->ja3 = all_flows[i].flow->ssh_ssl.ja3_client;
+                  newJA3->unsafe_cipher = all_flows[i].flow->ssh_ssl.client_unsafe_cipher;
+                  HASH_ADD_KEYPTR(hh, ja3ByHostFound->host_client_info_hasht, newJA3->ja3, strlen(newJA3->ja3), newJA3);
+               }
+            }
+
+            //ja3 -> host ip
+            HASH_FIND_STR(hostByJA3C_ht, all_flows[i].flow->ssh_ssl.ja3_client, hostByJA3Found);
+            if(hostByJA3Found == NULL){
+               ndpi_ip_dns *newHost = malloc(sizeof(ndpi_ip_dns));
+
+               newHost->ip = all_flows[i].flow->src_ip;
+               newHost->ip_string = all_flows[i].flow->src_name;
+               newHost->dns_name = all_flows[i].flow->ssh_ssl.client_info;;
+
+               ndpi_ja3_fingerprints_host *newElement = malloc(sizeof(ndpi_ja3_fingerprints_host));
+               newElement->ja3 = all_flows[i].flow->ssh_ssl.ja3_client;
+               newElement->unsafe_cipher = all_flows[i].flow->ssh_ssl.client_unsafe_cipher;
+               newElement->ipToDNS_ht = NULL;
+
+               HASH_ADD_INT(newElement->ipToDNS_ht, ip, newHost);
+               HASH_ADD_KEYPTR(hh, hostByJA3C_ht, newElement->ja3, strlen(newElement->ja3), newElement);
+            }else{
+               ndpi_ip_dns *innerElement = NULL;
+               HASH_FIND_INT(hostByJA3Found->ipToDNS_ht, &(all_flows[i].flow->src_ip), innerElement);
+               if(innerElement == NULL){
+                  ndpi_ip_dns *newInnerElement = malloc(sizeof(ndpi_ip_dns));
+                  newInnerElement->ip = all_flows[i].flow->src_ip;
+                  newInnerElement->ip_string = all_flows[i].flow->src_name;
+                  newInnerElement->dns_name = all_flows[i].flow->ssh_ssl.client_info;
+                  HASH_ADD_INT(hostByJA3Found->ipToDNS_ht, ip, newInnerElement);
+               }
+            }
+
+         }
+         if(all_flows[i].flow->ssh_ssl.ja3_server[0] != '\0'){
+            //looking if the host is already in the hash table
+            HASH_FIND_INT(ja3ByHostsHashT, &(all_flows[i].flow->dst_ip), ja3ByHostFound);
+            if(ja3ByHostFound == NULL){
+               //adding the new host in the hash table
+               ndpi_host_ja3_fingerprints *newHost = malloc(sizeof(ndpi_host_ja3_fingerprints));
+               newHost->host_client_info_hasht = NULL;
+               newHost->host_server_info_hasht = NULL;
+               newHost->ip_string = all_flows[i].flow->dst_name;
+               newHost->ip = all_flows[i].flow->dst_ip;
+               newHost->dns_name = all_flows[i].flow->ssh_ssl.server_info;
+
+               ndpi_ja3_info *newJA3 = malloc(sizeof(ndpi_ja3_info));
+               newJA3->ja3 = all_flows[i].flow->ssh_ssl.ja3_server;
+               newJA3->unsafe_cipher = all_flows[i].flow->ssh_ssl.server_unsafe_cipher;
+               //adding the new ja3 fingerprint
+               HASH_ADD_KEYPTR(hh, newHost->host_server_info_hasht, newJA3->ja3, strlen(newJA3->ja3), newJA3);
+               //adding the new host
+               HASH_ADD_INT(ja3ByHostsHashT, ip, newHost);
+            }else{
+               //host already in the hashtable
+               ndpi_ja3_info *infoFound = NULL;
+               HASH_FIND_STR(ja3ByHostFound->host_server_info_hasht, all_flows[i].flow->ssh_ssl.ja3_server, infoFound);
+               if(infoFound == NULL){
+                  ndpi_ja3_info *newJA3 = malloc(sizeof(ndpi_ja3_info));
+                  newJA3->ja3 = all_flows[i].flow->ssh_ssl.ja3_server;
+                  newJA3->unsafe_cipher = all_flows[i].flow->ssh_ssl.server_unsafe_cipher;
+                  HASH_ADD_KEYPTR(hh, ja3ByHostFound->host_server_info_hasht, newJA3->ja3, strlen(newJA3->ja3), newJA3);
+               }
+            }
+
+            HASH_FIND_STR(hostByJA3S_ht, all_flows[i].flow->ssh_ssl.ja3_server, hostByJA3Found);
+            if(hostByJA3Found == NULL){
+               ndpi_ip_dns *newHost = malloc(sizeof(ndpi_ip_dns));
+
+               newHost->ip = all_flows[i].flow->dst_ip;
+               newHost->ip_string = all_flows[i].flow->dst_name;
+               newHost->dns_name = all_flows[i].flow->ssh_ssl.server_info;;
+
+               ndpi_ja3_fingerprints_host *newElement = malloc(sizeof(ndpi_ja3_fingerprints_host));
+               newElement->ja3 = all_flows[i].flow->ssh_ssl.ja3_server;
+               newElement->unsafe_cipher = all_flows[i].flow->ssh_ssl.server_unsafe_cipher;
+               newElement->ipToDNS_ht = NULL;
+
+               HASH_ADD_INT(newElement->ipToDNS_ht, ip, newHost);
+               HASH_ADD_KEYPTR(hh, hostByJA3S_ht, newElement->ja3, strlen(newElement->ja3), newElement);
+            }else{
+               ndpi_ip_dns *innerElement = NULL;
+
+               HASH_FIND_INT(hostByJA3Found->ipToDNS_ht, &(all_flows[i].flow->dst_ip), innerElement);
+               if(innerElement == NULL){
+                  ndpi_ip_dns *newInnerElement = malloc(sizeof(ndpi_ip_dns));
+                  newInnerElement->ip = all_flows[i].flow->dst_ip;
+                  newInnerElement->ip_string = all_flows[i].flow->dst_name;
+                  newInnerElement->dns_name = all_flows[i].flow->ssh_ssl.server_info;
+                  HASH_ADD_INT(hostByJA3Found->ipToDNS_ht, ip, newInnerElement);
+               }
+            }
+
+         }
+      }
+
+      //pointers for the hash table's iteration
+      ndpi_host_ja3_fingerprints *ja3ByHost_element = NULL;
+      ndpi_ja3_info *info_of_element = NULL;
+      ndpi_host_ja3_fingerprints *tmp = NULL;
+      ndpi_ja3_info *tmp2 = NULL;
+
+      unsigned int num_ja3_client;
+      unsigned int num_ja3_server;
+
+      if(verbose == 1){
+         //for each host the number of flow with a ja3 fingerprint is printed
+         int i = 1;
+         ndpi_host_ja3_fingerprints *ja3ByHost_element;
+         printf("Number of flow with JA3 for each host: \n");
+         for(ja3ByHost_element = ja3ByHostsHashT; ja3ByHost_element != NULL; ja3ByHost_element = ja3ByHost_element->hh.next){
+            num_ja3_client = HASH_COUNT(ja3ByHost_element->host_client_info_hasht);
+            num_ja3_server = HASH_COUNT(ja3ByHost_element->host_server_info_hasht);
+
+            if(num_ja3_client > 0){
+               printf("\t%d\t IP %-42s \t %-10d JA3C\n",
+                  i,
+                  ja3ByHost_element->ip_string,
+                  num_ja3_client
+               );
+               i++;
+            }
+            else if(num_ja3_server > 0){
+               printf("\t%d\t IP %-42s \t %-10d JA3S %s\n",
+                  i,
+                  ja3ByHost_element->ip_string,
+                  num_ja3_server,
+                  ja3ByHost_element->dns_name
+               );
+               i++;
+            }
+         }
+      }else if(verbose == 2){
+         int i = 1;
+         int againstRepeat;
+         //for each host it is printted the JA3C and JA3S, along the server name (if any)
+         //and the security status
+
+         //first all clients, then all servers
+         printf("JA3C & JA3S for each host: \n");
+
+         //reminder
+            //ja3ByHostsHashT: hash table <ip, (ja3, ht_client, ht_server)>
+            //ja3ByHost_element: element of ja3ByHostsHashT
+            //info_of_element: element of the inner hash table of ja3ByHost_element
+         HASH_ITER(hh, ja3ByHostsHashT, ja3ByHost_element, tmp){
+            num_ja3_client = HASH_COUNT(ja3ByHost_element->host_client_info_hasht);
+            num_ja3_server = HASH_COUNT(ja3ByHost_element->host_server_info_hasht);
+            againstRepeat = 0;
+            if(num_ja3_client > 0){
+               HASH_ITER(hh, ja3ByHost_element->host_client_info_hasht, info_of_element, tmp2){
+                  if(againstRepeat == 0){
+                     printf("\t%-15d %-42s JA3C %s %s\n",
+                        i,
+                        ja3ByHost_element->ip_string,
+                        info_of_element->ja3,
+                        print_cipher(info_of_element->unsafe_cipher)
+                     );
+                     againstRepeat = 1;
+                     i++;
+                  }
+                  else{
+                     printf("\t%58s JA3C %s %s\n",
+                        "",
+                        info_of_element->ja3,
+                        print_cipher(info_of_element->unsafe_cipher)
+                     );
+                  }
+               }
+            }
+            againstRepeat = 0;
+            if(num_ja3_server > 0){
+               HASH_ITER(hh, ja3ByHost_element->host_server_info_hasht, info_of_element, tmp2){
+                  if(againstRepeat == 0){
+                     printf("\t%-15d %-42s JA3S %s %-10s %s\n",
+                        i,
+                        ja3ByHost_element->ip_string,
+                        info_of_element->ja3,
+                        print_cipher(info_of_element->unsafe_cipher),
+                        ja3ByHost_element->dns_name
+                     );
+                     againstRepeat = 1;
+                     i++;
+                  }else{
+                     printf("\t%58s JA3S %s %-10s %s\n",
+                        "",
+                        info_of_element->ja3,
+                        print_cipher(info_of_element->unsafe_cipher),
+                        ja3ByHost_element->dns_name
+                     );
+                  }
+               }
+            }
+         }
+
+         ndpi_ja3_fingerprints_host *hostByJA3Element = NULL;
+         ndpi_ja3_fingerprints_host *tmp3 = NULL;
+         ndpi_ip_dns *innerHashEl = NULL;
+         ndpi_ip_dns *tmp4 = NULL;
+
+         i = 1;
+         printf("\nHost IP for each JA3:\n");
+         HASH_ITER(hh, hostByJA3C_ht, hostByJA3Element, tmp3){
+            againstRepeat = 0;
+            HASH_ITER(hh, hostByJA3Element->ipToDNS_ht, innerHashEl, tmp4){
+               if(againstRepeat == 0){
+                  printf("\t%-15d JA3C %s",
+                     i,
+                     hostByJA3Element->ja3
+                  );
+                  printf("\t IP %-42s %s\n",
+                     innerHashEl->ip_string,
+                     print_cipher(hostByJA3Element->unsafe_cipher)
+                  );
+                  againstRepeat = 1;
+                  i++;
+               }else{
+                  printf("\t%51s", "");
+                  printf("\t IP %-42s %s\n",
+                     innerHashEl->ip_string,
+                     print_cipher(hostByJA3Element->unsafe_cipher)
+                  );
+               }
+            }
+         }
+         HASH_ITER(hh, hostByJA3S_ht, hostByJA3Element, tmp3){
+            againstRepeat = 0;
+            HASH_ITER(hh, hostByJA3Element->ipToDNS_ht, innerHashEl, tmp4){
+               if(againstRepeat == 0){
+                  printf("\t%-15d JA3S %s",
+                     i,
+                     hostByJA3Element->ja3
+                  );
+                  printf("\t IP %-42s %-10s %s\n",
+                     innerHashEl->ip_string,
+                     print_cipher(hostByJA3Element->unsafe_cipher),
+                     innerHashEl->dns_name
+                  );
+                  againstRepeat = 1;
+                  i++;
+               }else{
+                  printf("\t%51s", "");
+                  printf("\t IP %-42s %-10s %s\n",
+                     innerHashEl->ip_string,
+                     print_cipher(hostByJA3Element->unsafe_cipher),
+                     innerHashEl->dns_name
+                  );
+               }
+            }
+         }
+
+      }
+      printf("\n\n");
+
+      //freeing the hash table
+      HASH_ITER(hh, ja3ByHostsHashT, ja3ByHost_element, tmp){
+         HASH_ITER(hh, ja3ByHost_element->host_client_info_hasht, info_of_element, tmp2){
+            HASH_DEL(ja3ByHost_element->host_client_info_hasht, info_of_element);
+            free(info_of_element);
+         }
+         HASH_ITER(hh, ja3ByHost_element->host_server_info_hasht, info_of_element, tmp2){
+            HASH_DEL(ja3ByHost_element->host_server_info_hasht, info_of_element);
+            free(info_of_element);
+         }
+         HASH_DEL(ja3ByHostsHashT, ja3ByHost_element);
+         free(ja3ByHost_element);
+      }
+
+      ndpi_ja3_fingerprints_host *hostByJA3Element = NULL;
+      ndpi_ja3_fingerprints_host *tmp3 = NULL;
+      ndpi_ip_dns *innerHashEl = NULL;
+      ndpi_ip_dns *tmp4 = NULL;
+
+      HASH_ITER(hh, hostByJA3C_ht, hostByJA3Element, tmp3){
+         HASH_ITER(hh, hostByJA3C_ht->ipToDNS_ht, innerHashEl, tmp4){
+            HASH_DEL(hostByJA3Element->ipToDNS_ht, innerHashEl);
+            free(innerHashEl);
+         }
+         HASH_DEL(hostByJA3C_ht, hostByJA3Element);
+         free(hostByJA3Element);
+      }
+      hostByJA3Element = NULL;
+      HASH_ITER(hh, hostByJA3S_ht, hostByJA3Element, tmp3){
+         HASH_ITER(hh, hostByJA3S_ht->ipToDNS_ht, innerHashEl, tmp4){
+            HASH_DEL(hostByJA3Element->ipToDNS_ht, innerHashEl);
+            free(innerHashEl);
+         }
+         HASH_DEL(hostByJA3S_ht, hostByJA3Element);
+         free(hostByJA3Element);
+      }
+
     qsort(all_flows, num_flows, sizeof(struct flow_info), cmpFlows);
 
     for(i=0; i<num_flows; i++)
@@ -2458,7 +2789,7 @@ static void ndpi_process_packet(u_char *args,
   uint8_t *packet_checked = malloc(header->caplen);
 
   /* ndpi_has_human_readeable_string(ndpi_info_mod, (char*)packet, header->caplen, 3); */
-  
+
   memcpy(packet_checked, packet, header->caplen);
   p = ndpi_workflow_process_packet(ndpi_thread_info[thread_id].workflow, header, packet_checked);
 
diff --git a/example/ndpi_util.h b/example/ndpi_util.h
index dfcc98a72..696679674 100644
--- a/example/ndpi_util.h
+++ b/example/ndpi_util.h
@@ -29,6 +29,7 @@
 #ifndef __NDPI_UTIL_H__
 #define __NDPI_UTIL_H__
 
+#include "uthash.h"
 #include <pcap.h>
 
 #ifdef USE_DPDK
@@ -73,6 +74,45 @@ extern int dpdk_port_init(int port, struct rte_mempool *mbuf_pool);
 #define MAX_TABLE_SIZE_2         8192
 #define INIT_VAL                   -1
 
+
+// inner hash table (ja3 -> security state)
+typedef struct ndpi_ja3_info {
+   char * ja3;
+   ndpi_cipher_weakness unsafe_cipher;
+   UT_hash_handle hh;
+} ndpi_ja3_info;
+
+// external hash table (host ip -> <ip string, hash table ja3c, hash table ja3s>)
+// used to aggregate ja3 fingerprints by hosts
+typedef struct ndpi_host_ja3_fingerprints{
+   u_int32_t ip;
+   char *ip_string;
+   char *dns_name;
+   ndpi_ja3_info *host_client_info_hasht;
+   ndpi_ja3_info *host_server_info_hasht;
+
+   UT_hash_handle hh;
+} ndpi_host_ja3_fingerprints;
+
+
+//inner hash table
+typedef struct ndpi_ip_dns{
+   u_int32_t ip;
+   char *ip_string;
+   char *dns_name; //server name if any;
+   UT_hash_handle hh;
+} ndpi_ip_dns;
+
+//hash table ja3 -> <host, ip, security>, used to aggregate host by ja3 fingerprints
+typedef struct ndpi_ja3_fingerprints_host{
+   char *ja3; //key
+   ndpi_cipher_weakness unsafe_cipher;
+   ndpi_ip_dns *ipToDNS_ht;
+   UT_hash_handle hh;
+} ndpi_ja3_fingerprints_host;
+
+
+
 // flow tracking
 typedef struct ndpi_flow_info {
   u_int32_t hashval;
@@ -96,7 +136,7 @@ typedef struct ndpi_flow_info {
   char host_server_name[256];
   char bittorent_hash[41];
   char dhcp_fingerprint[48];
-  
+
   struct {
     u_int16_t ssl_version;
     char client_info[64], server_info[64], server_organization[64],