aboutsummaryrefslogtreecommitdiff
path: root/example
diff options
context:
space:
mode:
Diffstat (limited to 'example')
-rw-r--r--example/ndpiReader.c290
-rw-r--r--example/reader_util.c390
-rw-r--r--example/reader_util.h39
3 files changed, 618 insertions, 101 deletions
diff --git a/example/ndpiReader.c b/example/ndpiReader.c
index 32321aa94..708e330e1 100644
--- a/example/ndpiReader.c
+++ b/example/ndpiReader.c
@@ -81,8 +81,8 @@ static json_object *jArray_topStats;
static u_int8_t live_capture = 0;
static u_int8_t undetected_flows_deleted = 0;
/** User preferences **/
-u_int8_t enable_protocol_guess = 1;
-u_int8_t verbose = 0, json_flag = 0;
+u_int8_t enable_protocol_guess = 1, enable_payload_analyzer = 0;
+u_int8_t verbose = 0, json_flag = 0, enable_joy_stats = 0;
int nDPI_LogLevel = 0;
char *_debug_protocols = NULL;
static u_int8_t stats_flag = 0, bpf_filter_flag = 0;
@@ -90,6 +90,7 @@ static u_int8_t stats_flag = 0, bpf_filter_flag = 0;
static u_int8_t file_first_time = 1;
#endif
u_int8_t human_readeable_string_len = 5;
+u_int8_t max_num_udp_dissected_pkts = 16 /* 8 is enough for most protocols, Signal requires more */, max_num_tcp_dissected_pkts = 10;
static u_int32_t pcap_analysis_duration = (u_int32_t)-1;
static u_int16_t decode_tunnels = 0;
static u_int16_t num_loops = 1;
@@ -212,6 +213,8 @@ static int dpdk_port_id = 0, dpdk_run_capture = 1;
void test_lib(); /* Forward */
+extern void ndpi_report_payload_stats();
+
/* ********************************** */
#ifdef DEBUG_TRACE
@@ -225,6 +228,110 @@ FILE *trace = NULL;
*/
static void setupDetection(u_int16_t thread_id, pcap_t * pcap_handle);
+static void reduceBDbits(uint32_t *bd, unsigned int len) {
+ int mask = 0;
+ int shift = 0;
+ unsigned int i = 0;
+
+ for(i = 0; i < len; i++)
+ mask = mask | bd[i];
+
+ mask = mask >> 8;
+ for(i = 0; i < 24 && mask; i++) {
+ mask = mask >> 1;
+ if (mask == 0) {
+ shift = i+1;
+ break;
+ }
+ }
+
+ for(i = 0; i < len; i++)
+ bd[i] = bd[i] >> shift;
+}
+
+/**
+ * @brief Get flow byte distribution mean and variance
+ */
+static void
+flowGetBDMeanandVariance(struct ndpi_flow_info* flow) {
+ FILE *out = results_file ? results_file : stdout;
+
+ const uint32_t *array = NULL;
+ uint32_t tmp[256], i;
+ unsigned int num_bytes;
+ double mean = 0.0, variance = 0.0;
+
+ fflush(out);
+
+ /*
+ * Sum up the byte_count array for outbound and inbound flows,
+ * if this flow is bidirectional
+ */
+ if (!flow->bidirectional) {
+ array = flow->src2dst_byte_count;
+ num_bytes = flow->src2dst_l4_bytes;
+ for (i=0; i<256; i++) {
+ tmp[i] = flow->src2dst_byte_count[i];
+ }
+
+ if (flow->src2dst_num_bytes != 0) {
+ mean = flow->src2dst_bd_mean;
+ variance = flow->src2dst_bd_variance/(flow->src2dst_num_bytes - 1);
+ variance = sqrt(variance);
+
+ if (flow->src2dst_num_bytes == 1) {
+ variance = 0.0;
+ }
+ }
+ } else {
+ for (i=0; i<256; i++) {
+ tmp[i] = flow->src2dst_byte_count[i] + flow->dst2src_byte_count[i];
+ }
+ array = tmp;
+ num_bytes = flow->src2dst_l4_bytes + flow->dst2src_l4_bytes;
+
+ if (flow->src2dst_num_bytes + flow->dst2src_num_bytes != 0) {
+ mean = ((double)flow->src2dst_num_bytes)/((double)(flow->src2dst_num_bytes+flow->dst2src_num_bytes))*flow->src2dst_bd_mean +
+ ((double)flow->dst2src_num_bytes)/((double)(flow->dst2src_num_bytes+flow->src2dst_num_bytes))*flow->dst2src_bd_mean;
+
+ variance = ((double)flow->src2dst_num_bytes)/((double)(flow->src2dst_num_bytes+flow->dst2src_num_bytes))*flow->src2dst_bd_variance +
+ ((double)flow->dst2src_num_bytes)/((double)(flow->dst2src_num_bytes+flow->src2dst_num_bytes))*flow->dst2src_bd_variance;
+
+ variance = variance/((double)(flow->src2dst_num_bytes + flow->dst2src_num_bytes - 1));
+ variance = sqrt(variance);
+ if (flow->src2dst_num_bytes + flow->dst2src_num_bytes == 1) {
+ variance = 0.0;
+ }
+ }
+ }
+
+ if(enable_joy_stats) {
+ if(verbose > 1) {
+ reduceBDbits(tmp, 256);
+ array = tmp;
+
+ fprintf(out, " [byte_dist: ");
+ for(i = 0; i < 255; i++)
+ fprintf(out, "%u,", (unsigned char)array[i]);
+
+ fprintf(out, "%u]", (unsigned char)array[i]);
+ }
+
+ /* Output the mean */
+ if(num_bytes != 0) {
+ fprintf(out, "][byte_dist_mean: %f", mean);
+ fprintf(out, "][byte_dist_std: %f]", variance);
+ }
+
+ if(num_bytes != 0) {
+ double entropy = ndpi_flow_get_byte_count_entropy(array, num_bytes);
+
+ fprintf(out, "[entropy: %f]", entropy);
+ fprintf(out, "[total_entropy: %f]", entropy * num_bytes);
+ }
+ }
+}
+
/**
* @brief Print help instructions
*/
@@ -236,8 +343,9 @@ static void help(u_int long_help) {
"-i <file|device> "
#endif
"[-f <filter>][-s <duration>][-m <duration>]\n"
- " [-p <protos>][-l <loops> [-q][-d][-h][-e <len>][-t][-v <level>]\n"
- " [-n <threads>][-w <file>][-c <file>][-j <file>][-x <file>]\n\n"
+ " [-p <protos>][-l <loops> [-q][-d][-J][-h][-e <len>][-t][-v <level>]\n"
+ " [-n <threads>][-w <file>][-c <file>][-j <file>][-x <file>]\n"
+ " [-T <num>][-U <num>]\n\n"
"Usage:\n"
" -i <file.pcap|device> | Specify a pcap file/playlist to read packets from or a\n"
" | device for live capture (comma-separated list)\n"
@@ -255,7 +363,10 @@ static void help(u_int long_help) {
" -d | Disable protocol guess and use only DPI\n"
" -e <len> | Min human readeable string match len. Default %u\n"
" -q | Quiet mode\n"
+ " -J | Display flow SPLT (sequence of packet length and time)\n"
+ " | and BD (byte distribution). See https://github.com/cisco/joy\n"
" -t | Dissect GTP/TZSP tunnels\n"
+ " -P | Enable payload analysis\n"
" -r | Print nDPI version and git revision\n"
" -c <path> | Load custom categories from the specified file\n"
" -w <path> | Write test output on the specified file. This is useful for\n"
@@ -270,8 +381,13 @@ static void help(u_int long_help) {
" | >3 - full debug + dbg_proto = all\n"
" -b <file.json> | Specify a file to write port based diagnose statistics\n"
" -x <file.json> | Produce bpf filters for specified diagnose file. Use\n"
- " | this option only for .json files generated with -b flag.\n",
- human_readeable_string_len);
+ " | this option only for .json files generated with -b flag.\n"
+ " -T <num> | Max number of TCP processed packets before giving up [default: %u]\n"
+ " -U <num> | Max number of UDP processed packets before giving up [default: %u]\n"
+ ,
+ human_readeable_string_len,
+ max_num_tcp_dissected_pkts,
+ max_num_udp_dissected_pkts);
#ifndef WIN32
printf("\nExcap (wireshark) options:\n"
@@ -327,6 +443,8 @@ static struct option longopts[] = {
{ "version", no_argument, NULL, 'V'},
{ "help", no_argument, NULL, 'h'},
{ "json", required_argument, NULL, 'j'},
+ { "joy", required_argument, NULL, 'J'},
+ { "payload-analysis", required_argument, NULL, 'P'},
{ "result-path", required_argument, NULL, 'w'},
{ "quiet", no_argument, NULL, 'q'},
@@ -477,7 +595,8 @@ static void parseOptions(int argc, char **argv) {
}
#endif
- while((opt = getopt_long(argc, argv, "e:c:df:g:i:hp:l:s:tv:V:n:j:rp:w:q0123:456:7:89:m:b:x:", longopts, &option_idx)) != EOF) {
+ while((opt = getopt_long(argc, argv, "e:c:df:g:i:hp:Pl:s:tv:V:n:j:Jrp:w:q0123:456:7:89:m:b:x:T:U:",
+ longopts, &option_idx)) != EOF) {
#ifdef DEBUG_TRACE
if(trace) fprintf(trace, " #### -%c [%s] #### \n", opt, optarg ? optarg : "");
#endif
@@ -573,6 +692,14 @@ static void parseOptions(int argc, char **argv) {
help(1);
break;
+ case 'J':
+ enable_joy_stats = 1;
+ break;
+
+ case 'P':
+ enable_payload_analyzer = 1;
+ break;
+
case 'j':
#ifndef HAVE_JSON_C
printf("WARNING: this copy of ndpiReader has been compiled without json-c: JSON export disabled\n");
@@ -634,6 +761,16 @@ static void parseOptions(int argc, char **argv) {
_debug_protocols = strdup(optarg);
break;
+ case 'T':
+ max_num_tcp_dissected_pkts = atoi(optarg);
+ if(max_num_tcp_dissected_pkts < 3) max_num_tcp_dissected_pkts = 3;
+ break;
+
+ case 'U':
+ max_num_udp_dissected_pkts = atoi(optarg);
+ if(max_num_udp_dissected_pkts < 3) max_num_udp_dissected_pkts = 3;
+ break;
+
default:
help(0);
break;
@@ -721,12 +858,12 @@ static char* ipProto2Name(u_int16_t proto_id) {
/* ********************************** */
+#if 0
/**
* @brief A faster replacement for inet_ntoa().
*/
char* intoaV4(u_int32_t addr, char* buf, u_int16_t bufLen) {
- char *cp, *retStr;
- uint byte;
+ char *cp;
int n;
cp = &buf[bufLen];
@@ -734,7 +871,8 @@ char* intoaV4(u_int32_t addr, char* buf, u_int16_t bufLen) {
n = 4;
do {
- byte = addr & 0xff;
+ u_int byte = addr & 0xff;
+
*--cp = byte % 10 + '0';
byte /= 10;
if(byte > 0) {
@@ -743,15 +881,14 @@ char* intoaV4(u_int32_t addr, char* buf, u_int16_t bufLen) {
if(byte > 0)
*--cp = byte + '0';
}
- *--cp = '.';
+ if(n > 1)
+ *--cp = '.';
addr >>= 8;
- } while(--n > 0);
+ } while (--n > 0);
- /* Convert the string to lowercase */
- retStr = (char*)(cp+1);
-
- return(retStr);
+ return(cp);
}
+#endif
/* ********************************** */
@@ -799,6 +936,13 @@ static void printFlow(u_int16_t id, struct ndpi_flow_info *flow, u_int16_t threa
if(flow->vlan_id > 0) fprintf(out, "[VLAN: %u]", flow->vlan_id);
+ if(enable_joy_stats) {
+ /* Print entropy values for monitored flows. */
+ flowGetBDMeanandVariance(flow);
+ fflush(out);
+ fprintf(out, "[score: %.4f]", flow->score);
+ }
+
if(flow->detected_protocol.master_protocol) {
char buf[64];
@@ -826,15 +970,19 @@ static void printFlow(u_int16_t id, struct ndpi_flow_info *flow, u_int16_t threa
if(flow->info[0] != '\0') fprintf(out, "[%s]", flow->info);
- if(flow->ssh_ssl.ssl_version != 0) fprintf(out, "[%s]", ndpi_ssl_version2str(flow->ssh_ssl.ssl_version));
- if(flow->ssh_ssl.ja3_client[0] != '\0') fprintf(out, "[JA3C: %s%s]", flow->ssh_ssl.ja3_client,
- print_cipher(flow->ssh_ssl.client_unsafe_cipher));
- if(flow->ssh_ssl.server_info[0] != '\0') fprintf(out, "[server: %s]", flow->ssh_ssl.server_info);
-
- if(flow->ssh_ssl.ja3_server[0] != '\0') fprintf(out, "[JA3S: %s%s]", flow->ssh_ssl.ja3_server,
- print_cipher(flow->ssh_ssl.server_unsafe_cipher));
- if(flow->ssh_ssl.server_organization[0] != '\0') fprintf(out, "[organization: %s]", flow->ssh_ssl.server_organization);
- if(flow->ssh_ssl.server_cipher != '\0') fprintf(out, "[Cipher: %s]", ndpi_cipher2str(flow->ssh_ssl.server_cipher));
+ if(flow->ssh_tls.ssl_version != 0) fprintf(out, "[%s]", ndpi_ssl_version2str(flow->ssh_tls.ssl_version));
+ if(flow->ssh_tls.client_info[0] != '\0') fprintf(out, "[client: %s]", flow->ssh_tls.client_info);
+ if(flow->ssh_tls.client_hassh[0] != '\0') fprintf(out, "[HASSH-C: %s]", flow->ssh_tls.client_hassh);
+
+ if(flow->ssh_tls.ja3_client[0] != '\0') fprintf(out, "[JA3C: %s%s]", flow->ssh_tls.ja3_client,
+ print_cipher(flow->ssh_tls.client_unsafe_cipher));
+ if(flow->ssh_tls.server_info[0] != '\0') fprintf(out, "[server: %s]", flow->ssh_tls.server_info);
+ if(flow->ssh_tls.server_hassh[0] != '\0') fprintf(out, "[HASSH-S: %s]", flow->ssh_tls.server_hassh);
+
+ if(flow->ssh_tls.ja3_server[0] != '\0') fprintf(out, "[JA3S: %s%s]", flow->ssh_tls.ja3_server,
+ print_cipher(flow->ssh_tls.server_unsafe_cipher));
+ if(flow->ssh_tls.server_organization[0] != '\0') fprintf(out, "[organization: %s]", flow->ssh_tls.server_organization);
+ if(flow->ssh_tls.server_cipher != '\0') fprintf(out, "[Cipher: %s]", ndpi_cipher2str(flow->ssh_tls.server_cipher));
if(flow->bittorent_hash[0] != '\0') fprintf(out, "[BT Hash: %s]", flow->bittorent_hash);
if(flow->dhcp_fingerprint[0] != '\0') fprintf(out, "[DHCP Fingerprint: %s]", flow->dhcp_fingerprint);
@@ -881,25 +1029,25 @@ static void printFlow(u_int16_t id, struct ndpi_flow_info *flow, u_int16_t threa
if(flow->host_server_name[0] != '\0')
json_object_object_add(jObj,"host.server.name",json_object_new_string(flow->host_server_name));
- if((flow->ssh_ssl.client_info[0] != '\0') || (flow->ssh_ssl.server_info[0] != '\0')) {
+ if((flow->ssh_tls.client_info[0] != '\0') || (flow->ssh_tls.server_info[0] != '\0')) {
json_object *sjObj = json_object_new_object();
- if(flow->ssh_ssl.ja3_server[0] != '\0')
- json_object_object_add(jObj,"ja3s",json_object_new_string(flow->ssh_ssl.ja3_server));
+ if(flow->ssh_tls.ja3_server[0] != '\0')
+ json_object_object_add(jObj,"ja3s",json_object_new_string(flow->ssh_tls.ja3_server));
- if(flow->ssh_ssl.ja3_client[0] != '\0')
- json_object_object_add(jObj,"ja3c",json_object_new_string(flow->ssh_ssl.ja3_client));
+ if(flow->ssh_tls.ja3_client[0] != '\0')
+ json_object_object_add(jObj,"ja3c",json_object_new_string(flow->ssh_tls.ja3_client));
- if(flow->ssh_ssl.ja3_server[0] != '\0')
- json_object_object_add(jObj,"host.server.ja3",json_object_new_string(flow->ssh_ssl.ja3_server));
+ if(flow->ssh_tls.ja3_server[0] != '\0')
+ json_object_object_add(jObj,"host.server.ja3",json_object_new_string(flow->ssh_tls.ja3_server));
- if(flow->ssh_ssl.client_info[0] != '\0')
- json_object_object_add(sjObj, "client", json_object_new_string(flow->ssh_ssl.client_info));
+ if(flow->ssh_tls.client_info[0] != '\0')
+ json_object_object_add(sjObj, "client", json_object_new_string(flow->ssh_tls.client_info));
- if(flow->ssh_ssl.server_info[0] != '\0')
- json_object_object_add(sjObj, "server", json_object_new_string(flow->ssh_ssl.server_info));
+ if(flow->ssh_tls.server_info[0] != '\0')
+ json_object_object_add(sjObj, "server", json_object_new_string(flow->ssh_tls.server_info));
- json_object_object_add(jObj, "ssh_ssl", sjObj);
+ json_object_object_add(jObj, "ssh_tls", sjObj);
}
if(json_flag == 1)
@@ -1944,6 +2092,9 @@ void printPortStats(struct port_stats *stats) {
/* *********************************************** */
static void printFlowsStats() {
+ if(enable_payload_analyzer)
+ ndpi_report_payload_stats();
+
if(verbose) {
int thread_id;
FILE *out = results_file ? results_file : stdout;
@@ -1982,7 +2133,7 @@ static void printFlowsStats() {
ndpi_ja3_fingerprints_host *hostByJA3Found = NULL;
//check if this is a ssh-ssl flow
- if(all_flows[i].flow->ssh_ssl.ja3_client[0] != '\0'){
+ if(all_flows[i].flow->ssh_tls.ja3_client[0] != '\0'){
//looking if the host is already in the hash table
HASH_FIND_INT(ja3ByHostsHashT, &(all_flows[i].flow->src_ip), ja3ByHostFound);
@@ -1994,11 +2145,11 @@ static void printFlowsStats() {
newHost->host_server_info_hasht = NULL;
newHost->ip_string = all_flows[i].flow->src_name;
newHost->ip = all_flows[i].flow->src_ip;
- newHost->dns_name = all_flows[i].flow->ssh_ssl.client_info;
+ newHost->dns_name = all_flows[i].flow->ssh_tls.client_info;
ndpi_ja3_info *newJA3 = malloc(sizeof(ndpi_ja3_info));
- newJA3->ja3 = all_flows[i].flow->ssh_ssl.ja3_client;
- newJA3->unsafe_cipher = all_flows[i].flow->ssh_ssl.client_unsafe_cipher;
+ newJA3->ja3 = all_flows[i].flow->ssh_tls.ja3_client;
+ newJA3->unsafe_cipher = all_flows[i].flow->ssh_tls.client_unsafe_cipher;
//adding the new ja3 fingerprint
HASH_ADD_KEYPTR(hh, newHost->host_client_info_hasht,
newJA3->ja3, strlen(newJA3->ja3), newJA3);
@@ -2009,29 +2160,29 @@ static void printFlowsStats() {
ndpi_ja3_info *infoFound = NULL;
HASH_FIND_STR(ja3ByHostFound->host_client_info_hasht,
- all_flows[i].flow->ssh_ssl.ja3_client, infoFound);
+ all_flows[i].flow->ssh_tls.ja3_client, infoFound);
if(infoFound == NULL){
ndpi_ja3_info *newJA3 = malloc(sizeof(ndpi_ja3_info));
- newJA3->ja3 = all_flows[i].flow->ssh_ssl.ja3_client;
- newJA3->unsafe_cipher = all_flows[i].flow->ssh_ssl.client_unsafe_cipher;
+ newJA3->ja3 = all_flows[i].flow->ssh_tls.ja3_client;
+ newJA3->unsafe_cipher = all_flows[i].flow->ssh_tls.client_unsafe_cipher;
HASH_ADD_KEYPTR(hh, ja3ByHostFound->host_client_info_hasht,
newJA3->ja3, strlen(newJA3->ja3), newJA3);
}
}
//ja3 -> host ip
- HASH_FIND_STR(hostByJA3C_ht, all_flows[i].flow->ssh_ssl.ja3_client, hostByJA3Found);
+ HASH_FIND_STR(hostByJA3C_ht, all_flows[i].flow->ssh_tls.ja3_client, hostByJA3Found);
if(hostByJA3Found == NULL){
ndpi_ip_dns *newHost = malloc(sizeof(ndpi_ip_dns));
newHost->ip = all_flows[i].flow->src_ip;
newHost->ip_string = all_flows[i].flow->src_name;
- newHost->dns_name = all_flows[i].flow->ssh_ssl.client_info;;
+ newHost->dns_name = all_flows[i].flow->ssh_tls.client_info;;
ndpi_ja3_fingerprints_host *newElement = malloc(sizeof(ndpi_ja3_fingerprints_host));
- newElement->ja3 = all_flows[i].flow->ssh_ssl.ja3_client;
- newElement->unsafe_cipher = all_flows[i].flow->ssh_ssl.client_unsafe_cipher;
+ newElement->ja3 = all_flows[i].flow->ssh_tls.ja3_client;
+ newElement->unsafe_cipher = all_flows[i].flow->ssh_tls.client_unsafe_cipher;
newElement->ipToDNS_ht = NULL;
HASH_ADD_INT(newElement->ipToDNS_ht, ip, newHost);
@@ -2044,13 +2195,13 @@ static void printFlowsStats() {
ndpi_ip_dns *newInnerElement = malloc(sizeof(ndpi_ip_dns));
newInnerElement->ip = all_flows[i].flow->src_ip;
newInnerElement->ip_string = all_flows[i].flow->src_name;
- newInnerElement->dns_name = all_flows[i].flow->ssh_ssl.client_info;
+ newInnerElement->dns_name = all_flows[i].flow->ssh_tls.client_info;
HASH_ADD_INT(hostByJA3Found->ipToDNS_ht, ip, newInnerElement);
}
}
}
- if(all_flows[i].flow->ssh_ssl.ja3_server[0] != '\0'){
+ if(all_flows[i].flow->ssh_tls.ja3_server[0] != '\0'){
//looking if the host is already in the hash table
HASH_FIND_INT(ja3ByHostsHashT, &(all_flows[i].flow->dst_ip), ja3ByHostFound);
if(ja3ByHostFound == NULL){
@@ -2060,11 +2211,11 @@ static void printFlowsStats() {
newHost->host_server_info_hasht = NULL;
newHost->ip_string = all_flows[i].flow->dst_name;
newHost->ip = all_flows[i].flow->dst_ip;
- newHost->dns_name = all_flows[i].flow->ssh_ssl.server_info;
+ newHost->dns_name = all_flows[i].flow->ssh_tls.server_info;
ndpi_ja3_info *newJA3 = malloc(sizeof(ndpi_ja3_info));
- newJA3->ja3 = all_flows[i].flow->ssh_ssl.ja3_server;
- newJA3->unsafe_cipher = all_flows[i].flow->ssh_ssl.server_unsafe_cipher;
+ newJA3->ja3 = all_flows[i].flow->ssh_tls.ja3_server;
+ newJA3->unsafe_cipher = all_flows[i].flow->ssh_tls.server_unsafe_cipher;
//adding the new ja3 fingerprint
HASH_ADD_KEYPTR(hh, newHost->host_server_info_hasht, newJA3->ja3,
strlen(newJA3->ja3), newJA3);
@@ -2074,27 +2225,27 @@ static void printFlowsStats() {
//host already in the hashtable
ndpi_ja3_info *infoFound = NULL;
HASH_FIND_STR(ja3ByHostFound->host_server_info_hasht,
- all_flows[i].flow->ssh_ssl.ja3_server, infoFound);
+ all_flows[i].flow->ssh_tls.ja3_server, infoFound);
if(infoFound == NULL){
ndpi_ja3_info *newJA3 = malloc(sizeof(ndpi_ja3_info));
- newJA3->ja3 = all_flows[i].flow->ssh_ssl.ja3_server;
- newJA3->unsafe_cipher = all_flows[i].flow->ssh_ssl.server_unsafe_cipher;
+ newJA3->ja3 = all_flows[i].flow->ssh_tls.ja3_server;
+ newJA3->unsafe_cipher = all_flows[i].flow->ssh_tls.server_unsafe_cipher;
HASH_ADD_KEYPTR(hh, ja3ByHostFound->host_server_info_hasht,
newJA3->ja3, strlen(newJA3->ja3), newJA3);
}
}
- HASH_FIND_STR(hostByJA3S_ht, all_flows[i].flow->ssh_ssl.ja3_server, hostByJA3Found);
+ HASH_FIND_STR(hostByJA3S_ht, all_flows[i].flow->ssh_tls.ja3_server, hostByJA3Found);
if(hostByJA3Found == NULL){
ndpi_ip_dns *newHost = malloc(sizeof(ndpi_ip_dns));
newHost->ip = all_flows[i].flow->dst_ip;
newHost->ip_string = all_flows[i].flow->dst_name;
- newHost->dns_name = all_flows[i].flow->ssh_ssl.server_info;;
+ newHost->dns_name = all_flows[i].flow->ssh_tls.server_info;;
ndpi_ja3_fingerprints_host *newElement = malloc(sizeof(ndpi_ja3_fingerprints_host));
- newElement->ja3 = all_flows[i].flow->ssh_ssl.ja3_server;
- newElement->unsafe_cipher = all_flows[i].flow->ssh_ssl.server_unsafe_cipher;
+ newElement->ja3 = all_flows[i].flow->ssh_tls.ja3_server;
+ newElement->unsafe_cipher = all_flows[i].flow->ssh_tls.server_unsafe_cipher;
newElement->ipToDNS_ht = NULL;
HASH_ADD_INT(newElement->ipToDNS_ht, ip, newHost);
@@ -2108,7 +2259,7 @@ static void printFlowsStats() {
ndpi_ip_dns *newInnerElement = malloc(sizeof(ndpi_ip_dns));
newInnerElement->ip = all_flows[i].flow->dst_ip;
newInnerElement->ip_string = all_flows[i].flow->dst_name;
- newInnerElement->dns_name = all_flows[i].flow->ssh_ssl.server_info;
+ newInnerElement->dns_name = all_flows[i].flow->ssh_tls.server_info;
HASH_ADD_INT(hostByJA3Found->ipToDNS_ht, ip, newInnerElement);
}
}
@@ -3094,10 +3245,11 @@ void serializerUnitTest() {
assert(ndpi_serialize_uint32_string(&serializer, i, "Hello") != -1);
assert(ndpi_serialize_string_string(&serializer, kbuf, vbuf) != -1);
assert(ndpi_serialize_string_uint32(&serializer, kbuf, i*i) != -1);
+ assert(ndpi_serialize_string_float(&serializer, kbuf, (float)(i*i), "%f") != -1);
}
if(trace)
- printf("Serialization size: %u/%u\n", serializer.size_used, serializer.buffer_size);
+ printf("Serialization size: %u\n", ndpi_serializer_get_buffer_len(&serializer));
assert(ndpi_init_deserializer(&deserializer, &serializer) != -1);
@@ -3109,7 +3261,8 @@ void serializerUnitTest() {
else {
u_int32_t k32, v32;
ndpi_string ks, vs;
-
+ float vf;
+
switch(et) {
case ndpi_serialization_uint32_uint32:
assert(ndpi_deserialize_uint32_uint32(&deserializer, &k32, &v32) != -1);
@@ -3149,6 +3302,17 @@ void serializerUnitTest() {
}
break;
+ case ndpi_serialization_string_float:
+ assert(ndpi_deserialize_string_float(&deserializer, &ks, &vf) != -1);
+ if(trace) {
+ u_int8_t bkpk = ks.str[ks.str_len];
+
+ ks.str[ks.str_len] = '\0';
+ printf("%s=%f\n", ks.str, vf);
+ ks.str[ks.str_len] = bkpk;
+ }
+ break;
+
default:
printf("serializerUnitTest: unsupported type %u detected!\n", et);
return;
diff --git a/example/reader_util.c b/example/reader_util.c
index d0ec31930..b8e21cce8 100644
--- a/example/reader_util.c
+++ b/example/reader_util.c
@@ -34,6 +34,8 @@
#else
#include <unistd.h>
#include <netinet/in.h>
+#include <math.h>
+#include <float.h>
#endif
#ifndef ETH_P_IP
@@ -73,9 +75,158 @@
#include "ndpi_main.h"
#include "reader_util.h"
+#include "ndpi_classify.h"
-extern u_int8_t enable_protocol_guess;
+extern u_int8_t enable_protocol_guess, enable_joy_stats, enable_payload_analyzer;
extern u_int8_t verbose, human_readeable_string_len;
+extern u_int8_t max_num_udp_dissected_pkts /* 8 */, max_num_tcp_dissected_pkts /* 10 */;
+
+static u_int32_t flow_id = 0;
+
+/* ****************************************************** */
+
+struct payload_stats {
+ u_int8_t *pattern;
+ u_int8_t pattern_len;
+ u_int16_t num_occurrencies;
+ UT_hash_handle hh; /* makes this structure hashable */
+};
+
+
+struct payload_stats *pstats = NULL;
+u_int32_t max_num_packets_per_flow = 32;
+u_int32_t max_packet_payload_dissection = 32; /* Full payload */
+u_int16_t min_pattern_len = 4;
+u_int16_t max_pattern_len = 8;
+
+
+void ndpi_analyze_payload(struct ndpi_flow_info *flow,
+ u_int8_t src_to_dst_direction,
+ u_int8_t *payload,
+ u_int16_t payload_len) {
+ struct payload_stats *ret;
+ u_int i;
+
+#ifdef DEBUG_PAYLOAD
+ for(i=0; i<payload_len; i++)
+ printf("%c", isprint(payload[i]) ? payload[i] : '.');
+ printf("\n");
+#endif
+
+ HASH_FIND(hh, pstats, payload, payload_len, ret);
+ if(ret == NULL) {
+ if((ret = (struct payload_stats*)calloc(1, sizeof(struct payload_stats))) == NULL)
+ return; /* OOM */
+
+ if((ret->pattern = (u_int8_t*)malloc(payload_len)) == NULL) {
+ free(ret);
+ return;
+ }
+
+ memcpy(ret->pattern, payload, payload_len);
+ ret->pattern_len = payload_len;
+ ret->num_occurrencies = 1;
+
+ HASH_ADD(hh, pstats, pattern[0], payload_len, ret);
+
+#ifdef DEBUG_PAYLOAD
+ printf("Added element [total: %u]\n", HASH_COUNT(pstats));
+#endif
+ } else {
+ ret->num_occurrencies++;
+ // printf("==> %u\n", ret->num_occurrencies);
+ }
+}
+
+
+void ndpi_payload_analyzer(struct ndpi_flow_info *flow,
+ u_int8_t src_to_dst_direction,
+ u_int8_t *payload, u_int16_t payload_len) {
+ u_int16_t i, j;
+ u_int16_t scan_len = ndpi_min(max_packet_payload_dissection, payload_len);
+
+ if((flow->src2dst_pkt_count+flow->dst2src_pkt_count) < max_num_packets_per_flow) {
+#ifdef DEBUG_PAYLOAD
+ printf("[hashval: %u][proto: %u][vlan: %u][%s:%u <-> %s:%u][direction: %s][payload_len: %u]\n",
+ flow->hashval, flow->protocol, flow->vlan_id,
+ flow->src_name, flow->src_port,
+ flow->dst_name, flow->dst_port,
+ src_to_dst_direction ? "s2d" : "d2s",
+ payload_len);
+#endif
+ } else
+ return;
+
+ for(i=0; i<scan_len; i++) {
+ for(j=min_pattern_len; j <= max_pattern_len; j++) {
+ if((i+j) < payload_len) {
+ ndpi_analyze_payload(flow, src_to_dst_direction, &payload[i], j);
+ ndpi_analyze_payload(flow, src_to_dst_direction, &payload[i], j);
+ }
+ }
+ }
+}
+
+/* ***************************************************** */
+
+static int payload_stats_sort_asc(void *_a, void *_b) {
+ struct payload_stats *a = (struct payload_stats *)_a;
+ struct payload_stats *b = (struct payload_stats *)_b;
+
+ //return(a->num_occurrencies - b->num_occurrencies);
+ return(b->num_occurrencies - a->num_occurrencies);
+}
+
+/* ***************************************************** */
+
+void print_payload_stat(struct payload_stats *p) {
+ u_int i;
+
+ printf("\t[");
+
+ for(i=0; i<p->pattern_len; i++) {
+ printf("%c", isprint(p->pattern[i]) ? p->pattern[i] : '.');
+ }
+
+ printf("]");
+ for(; i<16; i++) printf(" ");
+ printf("[");
+
+ for(i=0; i<p->pattern_len; i++) {
+ printf("%s%02X", (i > 0) ? " " : "", isprint(p->pattern[i]) ? p->pattern[i] : '.');
+ }
+
+ printf("]");
+
+ for(; i<16; i++) printf(" ");
+ for(i=p->pattern_len; i<max_pattern_len; i++) printf(" ");
+
+ printf("[len: %u][num_occurrencies: %u]\n",
+ p->pattern_len, p->num_occurrencies);
+}
+
+/* ***************************************************** */
+
+void ndpi_report_payload_stats() {
+ struct payload_stats *p, *tmp;
+ u_int num = 0, max_num = 25;
+
+ printf("\n\nPayload Analysis\n");
+
+ HASH_SORT(pstats, payload_stats_sort_asc);
+
+ HASH_ITER(hh, pstats, p, tmp) {
+ if(num <= max_num)
+ print_payload_stat(p);
+
+ free(p->pattern);
+ HASH_DEL(pstats, p);
+ free(p);
+ num++;
+ }
+}
+
+
/* ***************************************************** */
@@ -273,6 +424,94 @@ int ndpi_workflow_node_cmp(const void *a, const void *b) {
return(0); /* notreached */
}
+/**
+ * \brief Update the byte count for the flow record.
+ * \param f Flow data
+ * \param x Data to use for update
+ * \param len Length of the data (in bytes)
+ * \return none
+ */
+static void
+ndpi_flow_update_byte_count(struct ndpi_flow_info *flow, const void *x,
+ unsigned int len, u_int8_t src_to_dst_direction) {
+ const unsigned char *data = x;
+ u_int32_t i;
+ u_int32_t current_count = 0;
+
+ /*
+ * implementation note: The spec says that 4000 octets is enough of a
+ * sample size to accurately reflect the byte distribution. Also, to avoid
+ * wrapping of the byte count at the 16-bit boundry, we stop counting once
+ * the 4000th octet has been seen for a flow.
+ */
+
+ /* octet count was already incremented before processing this payload */
+ if (src_to_dst_direction) {
+ current_count = flow->src2dst_l4_bytes - len;
+ } else {
+ current_count = flow->dst2src_l4_bytes - len;
+ }
+
+ if (current_count < ETTA_MIN_OCTETS) {
+ for (i=0; i<len; i++) {
+ if (src_to_dst_direction) {
+ flow->src2dst_byte_count[data[i]]++;
+ } else {
+ flow->dst2src_byte_count[data[i]]++;
+ }
+ current_count++;
+ if (current_count >= ETTA_MIN_OCTETS) {
+ break;
+ }
+ }
+ }
+}
+
+/**
+ * \brief Update the byte distribution mean for the flow record.
+ * \param f Flow record
+ * \param x Data to use for update
+ * \param len Length of the data (in bytes)
+ * \return none
+ */
+static void
+ndpi_flow_update_byte_dist_mean_var(ndpi_flow_info_t *flow, const void *x,
+ unsigned int len, u_int8_t src_to_dst_direction) {
+ const unsigned char *data = x;
+ double delta;
+ unsigned int i;
+
+ for (i=0; i<len; i++) {
+ if (src_to_dst_direction) {
+ flow->src2dst_num_bytes += 1;
+ delta = ((double)data[i] - flow->src2dst_bd_mean);
+ flow->src2dst_bd_mean += delta/((double)flow->src2dst_num_bytes);
+ flow->src2dst_bd_variance += delta*((double)data[i] - flow->src2dst_bd_mean);
+ } else {
+ flow->dst2src_num_bytes += 1;
+ delta = ((double)data[i] - flow->dst2src_bd_mean);
+ flow->dst2src_bd_mean += delta/((double)flow->dst2src_num_bytes);
+ flow->dst2src_bd_variance += delta*((double)data[i] - flow->dst2src_bd_mean);
+ }
+ }
+}
+
+float
+ndpi_flow_get_byte_count_entropy(const uint32_t byte_count[256],
+ unsigned int num_bytes)
+{
+ int i;
+ float tmp, sum = 0.0;
+
+ for (i=0; i<256; i++) {
+ tmp = (float) byte_count[i] / (float) num_bytes;
+ if (tmp > FLT_EPSILON) {
+ sum -= tmp * logf(tmp);
+ }
+ }
+ return sum / logf(2.0);
+}
+
/* ***************************************************** */
static void patchIPv6Address(char *str) {
@@ -309,11 +548,13 @@ static struct ndpi_flow_info *get_ndpi_flow_info(struct ndpi_workflow * workflow
u_int8_t *proto,
u_int8_t **payload,
u_int16_t *payload_len,
- u_int8_t *src_to_dst_direction) {
+ u_int8_t *src_to_dst_direction,
+ struct timeval when) {
u_int32_t idx, l4_offset, hashval;
struct ndpi_flow_info flow;
void *ret;
const u_int8_t *l3, *l4;
+ u_int32_t l4_data_len = 0XFEEDFACE;
/*
Note: to keep things simple (ndpiReader is just a demo app)
@@ -363,6 +604,7 @@ static struct ndpi_flow_info *get_ndpi_flow_info(struct ndpi_workflow * workflow
tcp_len = ndpi_min(4*(*tcph)->doff, l4_packet_len);
*payload = (u_int8_t*)&l4[tcp_len];
*payload_len = ndpi_max(0, l4_packet_len-4*(*tcph)->doff);
+ l4_data_len = l4_packet_len - sizeof(struct ndpi_tcphdr);
} else if(iph->protocol == IPPROTO_UDP && l4_packet_len >= 8) {
// udp
@@ -371,9 +613,11 @@ static struct ndpi_flow_info *get_ndpi_flow_info(struct ndpi_workflow * workflow
*sport = ntohs((*udph)->source), *dport = ntohs((*udph)->dest);
*payload = (u_int8_t*)&l4[sizeof(struct ndpi_udphdr)];
*payload_len = (l4_packet_len > sizeof(struct ndpi_udphdr)) ? l4_packet_len-sizeof(struct ndpi_udphdr) : 0;
+ l4_data_len = l4_packet_len - sizeof(struct ndpi_udphdr);
} else {
// non tcp/udp protocols
*sport = *dport = 0;
+ l4_data_len = 0;
}
flow.protocol = iph->protocol, flow.vlan_id = vlan_id;
@@ -418,6 +662,7 @@ static struct ndpi_flow_info *get_ndpi_flow_info(struct ndpi_workflow * workflow
workflow->num_allocated_flows++;
memset(newflow, 0, sizeof(struct ndpi_flow_info));
+ newflow->flow_id = flow_id++;
newflow->hashval = hashval;
newflow->protocol = iph->protocol, newflow->vlan_id = vlan_id;
newflow->src_ip = iph->saddr, newflow->dst_ip = iph->daddr;
@@ -459,7 +704,15 @@ static struct ndpi_flow_info *get_ndpi_flow_info(struct ndpi_workflow * workflow
workflow->stats.ndpi_flow_count++;
*src = newflow->src_id, *dst = newflow->dst_id;
-
+ newflow->src2dst_pkt_len[newflow->src2dst_pkt_count] = l4_packet_len;
+ newflow->src2dst_pkt_time[newflow->src2dst_pkt_count] = when;
+ if (newflow->src2dst_pkt_count == 0) {
+ newflow->src2dst_start = when;
+ }
+ newflow->src2dst_pkt_count++;
+ if (l4_data_len != 0XFEEDFACE) {
+ newflow->src2dst_opackets++;
+ }
return newflow;
}
} else {
@@ -485,6 +738,28 @@ static struct ndpi_flow_info *get_ndpi_flow_info(struct ndpi_workflow * workflow
else
*src = flow->dst_id, *dst = flow->src_id, *src_to_dst_direction = 0, flow->bidirectional = 1;
}
+ if (src_to_dst_direction) {
+ if (flow->src2dst_pkt_count < MAX_NUM_PKTS) {
+ flow->src2dst_pkt_len[flow->src2dst_pkt_count] = l4_packet_len;
+ flow->src2dst_pkt_time[flow->src2dst_pkt_count] = when;
+ flow->src2dst_pkt_count++;
+ }
+ if (l4_data_len != 0XFEEDFACE) {
+ flow->src2dst_opackets++;
+ }
+ } else {
+ if (flow->dst2src_pkt_count < MAX_NUM_PKTS) {
+ flow->dst2src_pkt_len[flow->dst2src_pkt_count] = l4_packet_len;
+ flow->dst2src_pkt_time[flow->dst2src_pkt_count] = when;
+ if (flow->dst2src_pkt_count == 0) {
+ flow->dst2src_start = when;
+ }
+ flow->dst2src_pkt_count++;
+ }
+ if (l4_data_len != 0XFEEDFACE) {
+ flow->dst2src_opackets++;
+ }
+ }
return flow;
}
}
@@ -503,7 +778,8 @@ static struct ndpi_flow_info *get_ndpi_flow_info6(struct ndpi_workflow * workflo
u_int8_t *proto,
u_int8_t **payload,
u_int16_t *payload_len,
- u_int8_t *src_to_dst_direction) {
+ u_int8_t *src_to_dst_direction,
+ struct timeval when) {
struct ndpi_iphdr iph;
memset(&iph, 0, sizeof(iph));
@@ -523,12 +799,35 @@ static struct ndpi_flow_info *get_ndpi_flow_info6(struct ndpi_workflow * workflo
ntohs(iph6->ip6_hdr.ip6_un1_plen),
tcph, udph, sport, dport,
src, dst, proto, payload,
- payload_len, src_to_dst_direction));
+ payload_len, src_to_dst_direction, when));
}
/* ****************************************************** */
void process_ndpi_collected_info(struct ndpi_workflow * workflow, struct ndpi_flow_info *flow) {
+
+ if(enable_joy_stats) {
+ /* Update SPLT scores. */
+
+ if(flow->bidirectional)
+ flow->score = ndpi_classify(flow->src2dst_pkt_len, flow->src2dst_pkt_time,
+ flow->dst2src_pkt_len, flow->dst2src_pkt_time,
+ flow->src2dst_start, flow->dst2src_start,
+ MAX_NUM_PKTS, flow->src_port, flow->dst_port,
+ flow->src2dst_packets, flow->dst2src_packets,
+ flow->src2dst_opackets, flow->dst2src_opackets,
+ flow->src2dst_l4_bytes, flow->dst2src_l4_bytes, 1,
+ flow->src2dst_byte_count, flow->dst2src_byte_count);
+ else
+ flow->score = ndpi_classify(flow->src2dst_pkt_len, flow->src2dst_pkt_time,
+ NULL, NULL, flow->src2dst_start, flow->src2dst_start,
+ MAX_NUM_PKTS, flow->src_port, flow->dst_port,
+ flow->src2dst_packets, 0,
+ flow->src2dst_opackets, 0,
+ flow->src2dst_l4_bytes, 0, 1,
+ flow->src2dst_byte_count, NULL);
+ }
+
if(!flow->ndpi_flow) return;
snprintf(flow->host_server_name, sizeof(flow->host_server_name), "%s",
@@ -559,27 +858,33 @@ void process_ndpi_collected_info(struct ndpi_workflow * workflow, struct ndpi_fl
if(flow->detected_protocol.app_protocol != NDPI_PROTOCOL_DNS) {
/* SSH */
if(flow->detected_protocol.app_protocol == NDPI_PROTOCOL_SSH) {
- snprintf(flow->ssh_ssl.client_info, sizeof(flow->ssh_ssl.client_info), "%s",
+ snprintf(flow->ssh_tls.client_info, sizeof(flow->ssh_tls.client_info), "%s",
flow->ndpi_flow->protos.ssh.client_signature);
- snprintf(flow->ssh_ssl.server_info, sizeof(flow->ssh_ssl.server_info), "%s",
+ snprintf(flow->ssh_tls.server_info, sizeof(flow->ssh_tls.server_info), "%s",
flow->ndpi_flow->protos.ssh.server_signature);
+ snprintf(flow->ssh_tls.client_hassh, sizeof(flow->ssh_tls.client_hassh), "%s",
+ flow->ndpi_flow->protos.ssh.hassh_client);
+ snprintf(flow->ssh_tls.server_hassh, sizeof(flow->ssh_tls.server_hassh), "%s",
+ flow->ndpi_flow->protos.ssh.hassh_server);
}
- /* SSL */
- else if((flow->detected_protocol.app_protocol == NDPI_PROTOCOL_SSL)
- || (flow->detected_protocol.master_protocol == NDPI_PROTOCOL_SSL)) {
- flow->ssh_ssl.ssl_version = flow->ndpi_flow->protos.stun_ssl.ssl.ssl_version;
- snprintf(flow->ssh_ssl.client_info, sizeof(flow->ssh_ssl.client_info), "%s",
+ /* TLS */
+ else if((flow->detected_protocol.app_protocol == NDPI_PROTOCOL_TLS)
+ || (flow->detected_protocol.master_protocol == NDPI_PROTOCOL_TLS)
+ || (flow->ndpi_flow->protos.stun_ssl.ssl.ja3_client[0] != '\0')
+ ) {
+ flow->ssh_tls.ssl_version = flow->ndpi_flow->protos.stun_ssl.ssl.ssl_version;
+ snprintf(flow->ssh_tls.client_info, sizeof(flow->ssh_tls.client_info), "%s",
flow->ndpi_flow->protos.stun_ssl.ssl.client_certificate);
- snprintf(flow->ssh_ssl.server_info, sizeof(flow->ssh_ssl.server_info), "%s",
+ snprintf(flow->ssh_tls.server_info, sizeof(flow->ssh_tls.server_info), "%s",
flow->ndpi_flow->protos.stun_ssl.ssl.server_certificate);
- snprintf(flow->ssh_ssl.server_organization, sizeof(flow->ssh_ssl.server_organization), "%s",
+ snprintf(flow->ssh_tls.server_organization, sizeof(flow->ssh_tls.server_organization), "%s",
flow->ndpi_flow->protos.stun_ssl.ssl.server_organization);
- snprintf(flow->ssh_ssl.ja3_client, sizeof(flow->ssh_ssl.ja3_client), "%s",
+ snprintf(flow->ssh_tls.ja3_client, sizeof(flow->ssh_tls.ja3_client), "%s",
flow->ndpi_flow->protos.stun_ssl.ssl.ja3_client);
- snprintf(flow->ssh_ssl.ja3_server, sizeof(flow->ssh_ssl.ja3_server), "%s",
+ snprintf(flow->ssh_tls.ja3_server, sizeof(flow->ssh_tls.ja3_server), "%s",
flow->ndpi_flow->protos.stun_ssl.ssl.ja3_server);
- flow->ssh_ssl.server_unsafe_cipher = flow->ndpi_flow->protos.stun_ssl.ssl.server_unsafe_cipher;
- flow->ssh_ssl.server_cipher = flow->ndpi_flow->protos.stun_ssl.ssl.server_cipher;
+ flow->ssh_tls.server_unsafe_cipher = flow->ndpi_flow->protos.stun_ssl.ssl.server_unsafe_cipher;
+ flow->ssh_tls.server_cipher = flow->ndpi_flow->protos.stun_ssl.ssl.server_cipher;
}
}
@@ -613,7 +918,8 @@ static struct ndpi_proto packet_processing(struct ndpi_workflow * workflow,
u_int16_t ip_offset,
u_int16_t ipsize, u_int16_t rawsize,
const struct pcap_pkthdr *header,
- const u_char *packet) {
+ const u_char *packet,
+ struct timeval when) {
struct ndpi_id_struct *src, *dst;
struct ndpi_flow_info *flow = NULL;
struct ndpi_flow_struct *ndpi_flow = NULL;
@@ -631,12 +937,12 @@ static struct ndpi_proto packet_processing(struct ndpi_workflow * workflow,
ntohs(iph->tot_len) - (iph->ihl * 4),
&tcph, &udph, &sport, &dport,
&src, &dst, &proto,
- &payload, &payload_len, &src_to_dst_direction);
+ &payload, &payload_len, &src_to_dst_direction, when);
else
flow = get_ndpi_flow_info6(workflow, vlan_id, iph6, ip_offset,
&tcph, &udph, &sport, &dport,
&src, &dst, &proto,
- &payload, &payload_len, &src_to_dst_direction);
+ &payload, &payload_len, &src_to_dst_direction, when);
if(flow != NULL) {
workflow->stats.ip_packet_count++;
@@ -644,22 +950,34 @@ static struct ndpi_proto packet_processing(struct ndpi_workflow * workflow,
workflow->stats.total_ip_bytes += rawsize;
ndpi_flow = flow->ndpi_flow;
- if(src_to_dst_direction)
+ if(src_to_dst_direction) {
flow->src2dst_packets++, flow->src2dst_bytes += rawsize;
- else
+ flow->src2dst_l4_bytes += payload_len;
+ } else {
flow->dst2src_packets++, flow->dst2src_bytes += rawsize;
+ flow->dst2src_l4_bytes += payload_len;
+ }
+
+ if(enable_payload_analyzer && (payload_len > 0))
+ ndpi_payload_analyzer(flow, src_to_dst_direction, payload, payload_len);
+
+ if(enable_joy_stats) {
+ /* Update BD, distribution and mean. */
+ ndpi_flow_update_byte_count(flow, payload, payload_len, src_to_dst_direction);
+ ndpi_flow_update_byte_dist_mean_var(flow, payload, payload_len, src_to_dst_direction);
+ }
flow->last_seen = time;
if(!flow->has_human_readeable_strings) {
u_int8_t skip = 0;
-
+
if((proto == IPPROTO_TCP)
&& (
- (flow->detected_protocol.app_protocol == NDPI_PROTOCOL_SSL)
- || (flow->detected_protocol.master_protocol == NDPI_PROTOCOL_SSL)
+ (flow->detected_protocol.app_protocol == NDPI_PROTOCOL_TLS)
+ || (flow->detected_protocol.master_protocol == NDPI_PROTOCOL_TLS)
|| (flow->detected_protocol.app_protocol == NDPI_PROTOCOL_SSH)
- || (flow->detected_protocol.master_protocol == NDPI_PROTOCOL_SSH))
+ || (flow->detected_protocol.master_protocol == NDPI_PROTOCOL_SSH))
) {
if((flow->src2dst_packets+flow->dst2src_packets) < 10 /* MIN_NUM_ENCRYPT_SKIP_PACKETS */)
skip = 1;
@@ -667,7 +985,7 @@ static struct ndpi_proto packet_processing(struct ndpi_workflow * workflow,
if(!skip) {
char outbuf[64] = { '\0' };
-
+
if(ndpi_has_human_readeable_string(workflow->ndpi_struct, (char*)packet, header->caplen,
human_readeable_string_len,
flow->human_readeable_string_buffer,
@@ -677,10 +995,10 @@ static struct ndpi_proto packet_processing(struct ndpi_workflow * workflow,
} else {
if((proto == IPPROTO_TCP)
&& (
- (flow->detected_protocol.app_protocol == NDPI_PROTOCOL_SSL)
- || (flow->detected_protocol.master_protocol == NDPI_PROTOCOL_SSL)
+ (flow->detected_protocol.app_protocol == NDPI_PROTOCOL_TLS)
+ || (flow->detected_protocol.master_protocol == NDPI_PROTOCOL_TLS)
|| (flow->detected_protocol.app_protocol == NDPI_PROTOCOL_SSH)
- || (flow->detected_protocol.master_protocol == NDPI_PROTOCOL_SSH))
+ || (flow->detected_protocol.master_protocol == NDPI_PROTOCOL_SSH))
)
flow->has_human_readeable_strings = 0;
}
@@ -691,8 +1009,8 @@ static struct ndpi_proto packet_processing(struct ndpi_workflow * workflow,
if(!flow->detection_completed) {
u_int enough_packets =
- (((proto == IPPROTO_UDP) && ((flow->src2dst_packets + flow->dst2src_packets) > 8))
- || ((proto == IPPROTO_TCP) && ((flow->src2dst_packets + flow->dst2src_packets) > 10))) ? 1 : 0;
+ (((proto == IPPROTO_UDP) && ((flow->src2dst_packets + flow->dst2src_packets) > max_num_udp_dissected_pkts))
+ || ((proto == IPPROTO_TCP) && ((flow->src2dst_packets + flow->dst2src_packets) > max_num_tcp_dissected_pkts))) ? 1 : 0;
flow->detected_protocol = ndpi_detection_process_packet(workflow->ndpi_struct, ndpi_flow,
iph ? (uint8_t *)iph : (uint8_t *)iph6,
@@ -700,7 +1018,7 @@ static struct ndpi_proto packet_processing(struct ndpi_workflow * workflow,
if(enough_packets || (flow->detected_protocol.app_protocol != NDPI_PROTOCOL_UNKNOWN)) {
if((!enough_packets)
- && (flow->detected_protocol.master_protocol == NDPI_PROTOCOL_SSL)
+ && (flow->detected_protocol.master_protocol == NDPI_PROTOCOL_TLS)
&& (flow->ndpi_flow->protos.stun_ssl.ssl.ja3_server[0] == '\0'))
; /* Wait for JA3S certificate */
else {
@@ -985,7 +1303,7 @@ iph_check:
proto = options[0];
ip_len += 8 * (options[1] + 1);
}
-
+
iph = NULL;
} else {
static u_int8_t ipv4_warning_used = 0;
@@ -1070,7 +1388,7 @@ iph_check:
/* process the packet */
return(packet_processing(workflow, time, vlan_id, iph, iph6,
ip_offset, header->caplen - ip_offset,
- header->caplen, header, packet));
+ header->caplen, header, packet, header->ts));
}
/* ********************************************************** */
diff --git a/example/reader_util.h b/example/reader_util.h
index b006fd8d3..cf6acc7ec 100644
--- a/example/reader_util.h
+++ b/example/reader_util.h
@@ -31,6 +31,7 @@
#include "uthash.h"
#include <pcap.h>
+#include "ndpi_classify.h"
#ifdef USE_DPDK
#include <rte_eal.h>
@@ -50,6 +51,14 @@
extern int dpdk_port_init(int port, struct rte_mempool *mbuf_pool);
#endif
+/* ETTA Spec defiintions for feature readiness */
+#define ETTA_MIN_PACKETS 10
+#define ETTA_MIN_OCTETS 4000
+/** maximum line length */
+#define LINEMAX 512
+#define MAX_BYTE_COUNT_ARRAY_LENGTH 256
+#define MAX_NUM_PKTS 100
+
#define MAX_NUM_READER_THREADS 16
#define IDLE_SCAN_PERIOD 10 /* msec (use TICK_RESOLUTION = 1000) */
#define MAX_IDLE_TIME 30000
@@ -115,6 +124,7 @@ typedef struct ndpi_ja3_fingerprints_host{
// flow tracking
typedef struct ndpi_flow_info {
+ u_int32_t flow_id;
u_int32_t hashval;
u_int32_t src_ip;
u_int32_t dst_ip;
@@ -128,6 +138,7 @@ typedef struct ndpi_flow_info {
u_int64_t last_seen;
u_int64_t src2dst_bytes, dst2src_bytes;
u_int32_t src2dst_packets, dst2src_packets;
+ u_int32_t src2dst_opackets, dst2src_opackets;
u_int32_t has_human_readeable_strings;
char human_readeable_string_buffer[32];
@@ -141,13 +152,36 @@ typedef struct ndpi_flow_info {
struct {
u_int16_t ssl_version;
- char client_info[64], server_info[64], server_organization[64],
+ char client_info[64], server_info[64],
+ client_hassh[33], server_hassh[33],
+ server_organization[64],
ja3_client[33], ja3_server[33];
u_int16_t server_cipher;
ndpi_cipher_weakness client_unsafe_cipher, server_unsafe_cipher;
- } ssh_ssl;
+ } ssh_tls;
void *src_id, *dst_id;
+
+ // Entropy fields
+ u_int16_t src2dst_pkt_len[MAX_NUM_PKTS]; /*!< array of packet appdata lengths */
+ struct timeval src2dst_pkt_time[MAX_NUM_PKTS]; /*!< array of arrival times */
+ u_int16_t dst2src_pkt_len[MAX_NUM_PKTS]; /*!< array of packet appdata lengths */
+ struct timeval dst2src_pkt_time[MAX_NUM_PKTS]; /*!< array of arrival times */
+ struct timeval src2dst_start; /*!< first packet arrival time */
+ struct timeval dst2src_start; /*!< first packet arrival time */
+ u_int16_t src2dst_pkt_count; /*!< packet counts */
+ u_int16_t dst2src_pkt_count; /*!< packet counts */
+ u_int32_t src2dst_l4_bytes; /*!< packet counts */
+ u_int32_t dst2src_l4_bytes; /*!< packet counts */
+ u_int32_t src2dst_byte_count[256]; /*!< number of occurences of each byte */
+ u_int32_t dst2src_byte_count[256]; /*!< number of occurences of each byte */
+ u_int32_t src2dst_num_bytes;
+ u_int32_t dst2src_num_bytes;
+ double src2dst_bd_mean;
+ double src2dst_bd_variance;
+ double dst2src_bd_mean;
+ double dst2src_bd_variance;
+ float score;
} ndpi_flow_info_t;
@@ -245,6 +279,7 @@ void process_ndpi_collected_info(struct ndpi_workflow * workflow, struct ndpi_fl
u_int32_t ethernet_crc32(const void* data, size_t n_bytes);
void ndpi_flow_info_freer(void *node);
const char* print_cipher_id(u_int32_t cipher);
+float ndpi_flow_get_byte_count_entropy(const uint32_t byte_count[256], unsigned int num_bytes);
extern int nDPI_LogLevel;
Powered by cgit, Themed by Ted Yin.