aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--example/ndpiReader.c35
-rw-r--r--example/reader_util.c10
-rw-r--r--example/reader_util.h4
-rw-r--r--src/include/ndpi_typedefs.h8
-rw-r--r--src/lib/ndpi_main.c69
5 files changed, 97 insertions, 29 deletions
diff --git a/example/ndpiReader.c b/example/ndpiReader.c
index 921727291..19136091d 100644
--- a/example/ndpiReader.c
+++ b/example/ndpiReader.c
@@ -96,6 +96,7 @@ u_int8_t verbose = 0, enable_flow_stats = 0;
int nDPI_LogLevel = 0;
char *_debug_protocols = NULL;
static u_int8_t stats_flag = 0;
+ndpi_init_prefs init_prefs = ndpi_no_prefs;
u_int8_t human_readeable_string_len = 5;
u_int8_t max_num_udp_dissected_pkts = 24 /* 8 is enough for most protocols, Signal and SnapchatCall require more */, max_num_tcp_dissected_pkts = 80 /* due to telnet */;
static u_int32_t pcap_analysis_duration = (u_int32_t)-1;
@@ -120,7 +121,7 @@ static time_t capture_until = 0;
static u_int32_t num_flows;
static struct ndpi_detection_module_struct *ndpi_info_mod = NULL;
-extern u_int8_t enable_doh_dot_detection, enable_ja3_plus;
+extern u_int8_t enable_doh_dot_detection;
extern u_int32_t max_num_packets_per_flow, max_packet_payload_dissection, max_num_reported_top_payloads;
extern u_int16_t min_pattern_len, max_pattern_len;
extern void ndpi_self_check_host_match(); /* Self check function */
@@ -310,7 +311,7 @@ void ndpiCheckHostStringMatch(char *testChar) {
if(!testChar)
return;
- ndpi_str = ndpi_init_detection_module(enable_ja3_plus ? ndpi_enable_ja3_plus : ndpi_no_prefs);
+ ndpi_str = ndpi_init_detection_module(init_prefs);
ndpi_finalize_initialization(ndpi_str);
// Display ALL Host strings ie host_match[] ?
@@ -440,7 +441,7 @@ static void help(u_int long_help) {
"-i <file|device> "
#endif
"[-f <filter>][-s <duration>][-m <duration>][-b <num bin clusters>]\n"
- " [-p <protos>][-l <loops> [-q][-d][-J][-h][-H][-D][-e <len>][-t][-v <level>]\n"
+ " [-p <protos>][-l <loops> [-q][-d][-J][-h][-H][-D][-e <len>][-E][-t][-v <level>]\n"
" [-n <threads>][-w <file>][-c <file>][-C <file>][-j <file>][-x <file>]\n"
" [-r <file>][-j <file>][-S <file>][-T <num>][-U <num>] [-x <domain>][-z]\n"
" [-a <mode>]\n\n"
@@ -467,6 +468,7 @@ static void help(u_int long_help) {
" | 2 - List known risks\n"
" -d | Disable protocol guess and use only DPI\n"
" -e <len> | Min human readeable string match len. Default %u\n"
+ " -E | Track flow payload\n"
" -q | Quiet mode\n"
" -F | Enable flow stats\n"
" -t | Dissect GTP/TZSP tunnels\n"
@@ -531,7 +533,7 @@ static void help(u_int long_help) {
NDPI_PROTOCOL_BITMASK all;
- ndpi_info_mod = ndpi_init_detection_module(ndpi_no_prefs);
+ ndpi_info_mod = ndpi_init_detection_module(init_prefs);
printf("\n\nnDPI supported protocols:\n");
printf("%3s %-22s %-10s %-8s %-12s %s\n",
"Id", "Protocol", "Layer_4", "Nw_Proto", "Breed", "Category");
@@ -659,7 +661,7 @@ void extcap_config() {
ndpi_proto_defaults_t *proto_defaults;
#endif
- ndpi_info_mod = ndpi_init_detection_module(ndpi_no_prefs);
+ ndpi_info_mod = ndpi_init_detection_module(init_prefs);
#if 0
ndpi_num_supported_protocols = ndpi_get_ndpi_num_supported_protocols(ndpi_info_mod);
proto_defaults = ndpi_get_proto_defaults(ndpi_info_mod);
@@ -806,7 +808,7 @@ static void parseOptions(int argc, char **argv) {
}
#endif
- while((opt = getopt_long(argc, argv, "a:Ab:e:c:C:dDf:g:i:Ij:k:K:S:hHp:pP:l:r:s:tu:v:V:n:Jrp:x:w:zq0123:456:7:89:m:T:U:",
+ while((opt = getopt_long(argc, argv, "a:Ab:e:Ec:C:dDf:g:i:Ij:k:K:S:hHp:pP:l:r:s:tu:v:V:n:Jrp:x:w:zq0123:456:7:89:m:T:U:",
longopts, &option_idx)) != EOF) {
#ifdef DEBUG_TRACE
if(trace) fprintf(trace, " #### Handling option -%c [%s] #### \n", opt, optarg ? optarg : "");
@@ -838,6 +840,10 @@ static void parseOptions(int argc, char **argv) {
human_readeable_string_len = atoi(optarg);
break;
+ case 'E':
+ init_prefs |= ndpi_track_flow_payload;
+ break;
+
case 'i':
case '3':
_pcap_file[0] = optarg;
@@ -1053,7 +1059,7 @@ static void parseOptions(int argc, char **argv) {
break;
case 'z':
- enable_ja3_plus = 1;
+ init_prefs |= ndpi_enable_ja3_plus;
break;
default:
@@ -1678,6 +1684,17 @@ static void printFlow(u_int32_t id, struct ndpi_flow_info *flow, u_int16_t threa
print_bin(out, "Plen Bins", &flow->payload_len_bin);
#endif
+ if(flow->flow_payload && (flow->flow_payload_len > 0)) {
+ u_int i;
+
+ printf("[Payload: ");
+
+ for(i=0; i<flow->flow_payload_len; i++)
+ printf("%c", isspace(flow->flow_payload[i]) ? '.' : flow->flow_payload[i]);
+
+ printf("]");
+ }
+
fprintf(out, "\n");
}
}
@@ -4336,7 +4353,7 @@ static void dgaUnitTest() {
};
int debug = 0, i;
NDPI_PROTOCOL_BITMASK all;
- struct ndpi_detection_module_struct *ndpi_str = ndpi_init_detection_module(enable_ja3_plus ? ndpi_enable_ja3_plus : ndpi_no_prefs);
+ struct ndpi_detection_module_struct *ndpi_str = ndpi_init_detection_module(init_prefs);
assert(ndpi_str != NULL);
@@ -5097,7 +5114,7 @@ void zscoreUnitTest() {
ac_automata_enable_debug(1);
parseOptions(argc, argv);
- ndpi_info_mod = ndpi_init_detection_module(enable_ja3_plus ? ndpi_enable_ja3_plus : ndpi_no_prefs);
+ ndpi_info_mod = ndpi_init_detection_module(init_prefs);
if(ndpi_info_mod == NULL) return -1;
diff --git a/example/reader_util.c b/example/reader_util.c
index 7c19b31cb..498c834f8 100644
--- a/example/reader_util.c
+++ b/example/reader_util.c
@@ -78,7 +78,7 @@ extern u_int8_t max_num_udp_dissected_pkts /* 24 */, max_num_tcp_dissected_pkts
static u_int32_t flow_id = 0;
u_int8_t enable_doh_dot_detection = 0;
-u_int8_t enable_ja3_plus = 0;
+extern ndpi_init_prefs init_prefs;
/* ****************************************************** */
@@ -403,7 +403,7 @@ struct ndpi_workflow* ndpi_workflow_init(const struct ndpi_workflow_prefs * pref
set_ndpi_flow_malloc(NULL), set_ndpi_flow_free(NULL);
/* TODO: just needed here to init ndpi ndpi_malloc wrapper */
- module = ndpi_init_detection_module(enable_ja3_plus ? ndpi_enable_ja3_plus : ndpi_no_prefs);
+ module = ndpi_init_detection_module(init_prefs);
if(module == NULL) {
LOG(NDPI_LOG_ERROR, "global structure initialization failed\n");
@@ -537,7 +537,8 @@ void ndpi_flow_info_free_data(struct ndpi_flow_info *flow) {
ndpi_free_bin(&flow->payload_len_bin);
#endif
- if(flow->risk_str) ndpi_free(flow->risk_str);
+ if(flow->risk_str) ndpi_free(flow->risk_str);
+ if(flow->flow_payload) ndpi_free(flow->flow_payload);
}
/* ***************************************************** */
@@ -1329,6 +1330,9 @@ void process_ndpi_collected_info(struct ndpi_workflow * workflow, struct ndpi_fl
if(workflow->__flow_detected_callback != NULL)
workflow->__flow_detected_callback(workflow, flow, workflow->__flow_detected_udata);
}
+
+ flow->flow_payload = flow->ndpi_flow->flow_payload, flow->flow_payload_len = flow->ndpi_flow->flow_payload_len;
+ flow->ndpi_flow->flow_payload = NULL; /* We'll free the memory */
ndpi_free_flow_info_half(flow);
}
diff --git a/example/reader_util.h b/example/reader_util.h
index 703e33094..6a9c7cd40 100644
--- a/example/reader_util.h
+++ b/example/reader_util.h
@@ -300,6 +300,10 @@ typedef struct ndpi_flow_info {
#else
struct ndpi_bin payload_len_bin;
#endif
+
+ /* Flow payload */
+ u_int16_t flow_payload_len;
+ char *flow_payload;
} ndpi_flow_info_t;
diff --git a/src/include/ndpi_typedefs.h b/src/include/ndpi_typedefs.h
index 8d7f1e864..a8b270b6d 100644
--- a/src/include/ndpi_typedefs.h
+++ b/src/include/ndpi_typedefs.h
@@ -650,7 +650,6 @@ struct ndpi_lru_cache {
/* ************************************************** */
struct ndpi_flow_tcp_struct {
-
/* NDPI_PROTOCOL_MAIL_SMTP */
/* NDPI_PROTOCOL_MAIL_POP */
/* NDPI_PROTOCOL_MAIL_IMAP */
@@ -1242,6 +1241,8 @@ struct ndpi_detection_module_struct {
u_int8_t num_nbpf_custom_proto;
nbpf_filter nbpf_custom_proto[MAX_NBPF_CUSTOM_PROTO];
#endif
+
+ u_int16_t max_payload_track_len;
};
#endif /* NDPI_LIB_COMPILATION */
@@ -1554,6 +1555,10 @@ struct ndpi_flow_struct {
/* NDPI_PROTOCOL_TINC */
u_int8_t tinc_state;
+ /* Flow payload */
+ u_int16_t flow_payload_len;
+ char *flow_payload;
+
/*
Leave this field below at the end
The field below can be used by third
@@ -1619,6 +1624,7 @@ typedef enum {
ndpi_dont_load_icloud_private_relay_list = (1 << 13),
ndpi_dont_init_risk_ptree = (1 << 14),
ndpi_dont_load_cachefly_list = (1 << 15),
+ ndpi_track_flow_payload = (1 << 16),
} ndpi_prefs;
typedef struct {
diff --git a/src/lib/ndpi_main.c b/src/lib/ndpi_main.c
index b43cfd964..dca58a29a 100644
--- a/src/lib/ndpi_main.c
+++ b/src/lib/ndpi_main.c
@@ -2605,32 +2605,44 @@ struct ndpi_detection_module_struct *ndpi_init_detection_module(ndpi_init_prefs
if((ndpi_str->protocols_ptree = ndpi_patricia_new(32 /* IPv4 */)) != NULL) {
ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, host_protocol_list);
+
if(!(prefs & ndpi_dont_load_cachefly_list))
ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_cachefly_protocol_list);
+
if(!(prefs & ndpi_dont_load_tor_list))
ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_tor_protocol_list);
+
if(!(prefs & ndpi_dont_load_azure_list))
ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_microsoft_azure_protocol_list);
+
if(!(prefs & ndpi_dont_load_whatsapp_list))
ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_whatsapp_protocol_list);
+
if(!(prefs & ndpi_dont_load_amazon_aws_list))
ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_amazon_aws_protocol_list);
+
if(!(prefs & ndpi_dont_load_ethereum_list))
ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_mining_protocol_list);
+
if(!(prefs & ndpi_dont_load_zoom_list))
ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_zoom_protocol_list);
+
if(!(prefs & ndpi_dont_load_cloudflare_list))
ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_cloudflare_protocol_list);
+
if(!(prefs & ndpi_dont_load_microsoft_list)) {
ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_microsoft_365_protocol_list);
ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_ms_one_drive_protocol_list);
ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_ms_outlook_protocol_list);
ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_skype_teams_protocol_list);
}
+
if(!(prefs & ndpi_dont_load_google_list))
ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_google_protocol_list);
+
if(!(prefs & ndpi_dont_load_google_cloud_list))
ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_google_cloud_protocol_list);
+
if(!(prefs & ndpi_dont_load_asn_lists)) {
ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_telegram_protocol_list);
ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_apple_protocol_list);
@@ -2659,6 +2671,9 @@ struct ndpi_detection_module_struct *ndpi_init_detection_module(ndpi_init_prefs
ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_discord_protocol_list);
ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_line_protocol_list);
}
+
+ if(prefs & ndpi_track_flow_payload)
+ ndpi_str->max_payload_track_len = 1024; /* track up to X payload bytes */
}
ndpi_str->ip_risk_mask_ptree = ndpi_patricia_new(32 /* IPv4 */);
@@ -3137,7 +3152,7 @@ void ndpi_exit_detection_module(struct ndpi_detection_module_struct *ndpi_str) {
for(i = 0; (i < MAX_NBPF_CUSTOM_PROTO) && (ndpi_str->nbpf_custom_proto[i].tree != NULL); i++)
nbpf_free(ndpi_str->nbpf_custom_proto[i].tree);
#endif
-
+
/* NDPI_PROTOCOL_TINC */
if(ndpi_str->tinc_cache)
cache_free((cache_t)(ndpi_str->tinc_cache));
@@ -3660,10 +3675,10 @@ int ndpi_handle_rule(struct ndpi_detection_module_struct *ndpi_str, char *rule,
NDPI_LOG_ERR(ndpi_str, "nBPF: too many protocols\n");
return(-4); /* Too many protocols */
}
-
+
if(filter[0] == '"') {
u_int len;
-
+
filter = &filter[1];
len = strlen(filter);
@@ -4683,20 +4698,21 @@ static int ndpi_callback_init(struct ndpi_detection_module_struct *ndpi_str) {
/* Resize callback_buffer */
all_cb = ndpi_calloc(a+1,sizeof(struct ndpi_call_function_struct));
if(all_cb) {
- memcpy((char *)all_cb,(char *)ndpi_str->callback_buffer, (a+1) * sizeof(struct ndpi_call_function_struct));
- ndpi_free(ndpi_str->callback_buffer);
- ndpi_str->callback_buffer = all_cb;
+ memcpy((char *)all_cb,(char *)ndpi_str->callback_buffer, (a+1) * sizeof(struct ndpi_call_function_struct));
+ ndpi_free(ndpi_str->callback_buffer);
+ ndpi_str->callback_buffer = all_cb;
}
NDPI_LOG_DBG2(ndpi_str, "callback_buffer_size is %u\n", ndpi_str->callback_buffer_size);
/* Calculating the size of an array for callback functions */
ndpi_enabled_callbacks_init(ndpi_str,detection_bitmask,1);
all_cb = ndpi_calloc(ndpi_str->callback_buffer_size_tcp_payload +
- ndpi_str->callback_buffer_size_tcp_no_payload +
- ndpi_str->callback_buffer_size_udp +
- ndpi_str->callback_buffer_size_non_tcp_udp,
+ ndpi_str->callback_buffer_size_tcp_no_payload +
+ ndpi_str->callback_buffer_size_udp +
+ ndpi_str->callback_buffer_size_non_tcp_udp,
sizeof(struct ndpi_call_function_struct));
if(!all_cb) return 1;
+
ndpi_str->callback_buffer_tcp_payload = all_cb;
all_cb += ndpi_str->callback_buffer_size_tcp_payload;
ndpi_str->callback_buffer_tcp_no_payload = all_cb;
@@ -5063,8 +5079,13 @@ void ndpi_free_flow_data(struct ndpi_flow_struct* flow) {
}
}
}
+
+ if(flow->flow_payload != NULL)
+ ndpi_free(flow->flow_payload);
}
+/* ************************************************ */
+
void ndpi_set_protocol_detection_bitmask2(struct ndpi_detection_module_struct *ndpi_str,
const NDPI_PROTOCOL_BITMASK *dbm) {
ndpi_enabled_callbacks_init(ndpi_str,dbm,0);
@@ -5256,6 +5277,23 @@ void ndpi_connection_tracking(struct ndpi_detection_module_struct *ndpi_str,
const struct ndpi_tcphdr *tcph = packet->tcp;
const struct ndpi_udphdr *udph = packet->udp;
+ if(packet->payload_packet_len > 0) {
+ /* printf("LEN: %u [%s]\n", packet->payload_packet_len, packet->payload); */
+
+ if(flow->flow_payload == NULL)
+ flow->flow_payload = (char*)ndpi_malloc(ndpi_str->max_payload_track_len + 1);
+
+ if(flow->flow_payload != NULL) {
+ u_int i;
+
+ for(i=0; (i<packet->payload_packet_len)
+ && (flow->flow_payload_len < ndpi_str->max_payload_track_len); i++) {
+ flow->flow_payload[flow->flow_payload_len++] =
+ (isprint(packet->payload[i]) || isspace(packet->payload[i])) ? packet->payload[i] : '.';
+ }
+ }
+ }
+
packet->tcp_retransmission = 0, packet->packet_direction = 0;
if(ndpi_str->direction_detect_disable) {
@@ -5669,12 +5707,12 @@ static void ndpi_reconcile_protocols(struct ndpi_detection_module_struct *ndpi_s
/* Remove NDPI_UNIDIRECTIONAL_TRAFFIC from unidirectional protocols */
ndpi_unset_risk(ndpi_str, flow, NDPI_UNIDIRECTIONAL_TRAFFIC);
break;
-
+
case NDPI_PROTOCOL_SYSLOG:
if(flow->l4_proto == IPPROTO_UDP)
- ndpi_unset_risk(ndpi_str, flow, NDPI_UNIDIRECTIONAL_TRAFFIC);
+ ndpi_unset_risk(ndpi_str, flow, NDPI_UNIDIRECTIONAL_TRAFFIC);
break;
-
+
case NDPI_PROTOCOL_SKYPE_TEAMS:
case NDPI_PROTOCOL_SKYPE_TEAMS_CALL:
if(flow->is_ipv6 == 0
@@ -6362,8 +6400,8 @@ ndpi_protocol ndpi_detection_process_packet(struct ndpi_detection_module_struct
if(packet->tcp)
t.tuple.l4_src_port = packet->tcp->source, t.tuple.l4_dst_port = packet->tcp->dest;
else if(packet->udp)
- t.tuple.l4_src_port = packet->udp->source, t.tuple.l4_dst_port = packet->udp->dest;
-
+ t.tuple.l4_src_port = packet->udp->source, t.tuple.l4_dst_port = packet->udp->dest;
+
for(i=0; (i<MAX_NBPF_CUSTOM_PROTO) && (ndpi_str->nbpf_custom_proto[i].tree != NULL); i++) {
if(nbpf_match(ndpi_str->nbpf_custom_proto[i].tree, &t)) {
/* match found */
@@ -6371,14 +6409,13 @@ ndpi_protocol ndpi_detection_process_packet(struct ndpi_detection_module_struct
ndpi_fill_protocol_category(ndpi_str, flow, &ret);
ndpi_reconcile_protocols(ndpi_str, flow, &ret);
flow->confidence = NDPI_CONFIDENCE_NBPF;
-
+
return(ret);
}
}
}
#endif
-
ndpi_connection_tracking(ndpi_str, flow);
/* build ndpi_selection packet bitmask */
Powered by cgit, Themed by Ted Yin.