diff options
Diffstat (limited to 'src/lib')
| -rw-r--r-- | src/lib/ndpi_content_match.c.inc | 17 | ||||
| -rw-r--r-- | src/lib/ndpi_main.c | 179 | ||||
| -rw-r--r-- | src/lib/ndpi_serializer.c | 917 | ||||
| -rw-r--r-- | src/lib/ndpi_utils.c | 127 | ||||
| -rw-r--r-- | src/lib/protocols/bittorrent.c | 24 | ||||
| -rw-r--r-- | src/lib/protocols/http.c | 225 | ||||
| -rw-r--r-- | src/lib/protocols/kerberos.c | 46 | ||||
| -rw-r--r-- | src/lib/protocols/spotify.c | 2 | ||||
| -rw-r--r-- | src/lib/protocols/stun.c | 2 | ||||
| -rw-r--r-- | src/lib/protocols/tls.c | 185 |
10 files changed, 1168 insertions, 556 deletions
diff --git a/src/lib/ndpi_content_match.c.inc b/src/lib/ndpi_content_match.c.inc index 4b6daa50b..32a6e840c 100644 --- a/src/lib/ndpi_content_match.c.inc +++ b/src/lib/ndpi_content_match.c.inc @@ -8390,8 +8390,11 @@ static ndpi_network host_protocol_list[] = { { 0xD0163900 /* 208.22.57.0/24 */, 24, NDPI_PROTOCOL_BLOOMBERG }, { 0x45BFC000 /* 69.191.192.0/18 */, 18, NDPI_PROTOCOL_BLOOMBERG }, - /* Microsoft - https://docs.microsoft.com/en-us/office365/enterprise/urls-and-ip-address-ranges + /* + Microsoft + + [JSON] https://endpoints.office.com/endpoints/worldwide?clientrequestid=b10c5ed1-bad1-445f-b386-b919946339a7 + [HTML] https://docs.microsoft.com/en-us/office365/enterprise/urls-and-ip-address-ranges */ { 0x0D6B0698 /* 13.107.6.152/31 */, 31, NDPI_PROTOCOL_MICROSOFT_365 }, { 0x0D6B120A /* 13.107.18.10/31 */, 31, NDPI_PROTOCOL_MICROSOFT_365 }, @@ -8411,7 +8414,6 @@ static ndpi_network host_protocol_list[] = { { 0x1767A000 /* 23.103.160.0/20 */, 20, NDPI_PROTOCOL_MICROSOFT_365 }, { 0x28600000 /* 40.96.0.0/13 */, 13, NDPI_PROTOCOL_MICROSOFT_365 }, { 0x28680000 /* 40.104.0.0/15 */, 15, NDPI_PROTOCOL_MICROSOFT_365 }, - { 0x34600000 /* 52.96.0.0/14 */, 14, NDPI_PROTOCOL_MICROSOFT_365 }, { 0x83FD21D7 /* 131.253.33.215/32 */, 32, NDPI_PROTOCOL_MICROSOFT_365 }, { 0x84F50000 /* 132.245.0.0/16 */, 16, NDPI_PROTOCOL_MICROSOFT_365 }, { 0x96AB2000 /* 150.171.32.0/22 */, 22, NDPI_PROTOCOL_MICROSOFT_365 }, @@ -8423,7 +8425,6 @@ static ndpi_network host_protocol_list[] = { { 0x1767A000 /* 23.103.160.0/20 */, 20, NDPI_PROTOCOL_MICROSOFT_365 }, { 0x28600000 /* 40.96.0.0/13 */, 13, NDPI_PROTOCOL_MICROSOFT_365 }, { 0x28680000 /* 40.104.0.0/15 */, 15, NDPI_PROTOCOL_MICROSOFT_365 }, - { 0x34600000 /* 52.96.0.0/14 */, 14, NDPI_PROTOCOL_MICROSOFT_365 }, { 0x83FD21D7 /* 131.253.33.215/32 */, 32, NDPI_PROTOCOL_MICROSOFT_365 }, { 0x84F50000 /* 132.245.0.0/16 */, 16, NDPI_PROTOCOL_MICROSOFT_365 }, { 0x96AB2000 /* 150.171.32.0/22 */, 22, NDPI_PROTOCOL_MICROSOFT_365 }, @@ -8435,7 +8436,6 @@ static ndpi_network host_protocol_list[] = { { 0x1767A000 /* 23.103.160.0/20 */, 20, NDPI_PROTOCOL_MICROSOFT_365 }, { 0x28600000 /* 40.96.0.0/13 */, 13, NDPI_PROTOCOL_MICROSOFT_365 }, { 0x28680000 /* 40.104.0.0/15 */, 15, NDPI_PROTOCOL_MICROSOFT_365 }, - { 0x34600000 /* 52.96.0.0/14 */, 14, NDPI_PROTOCOL_MICROSOFT_365 }, { 0x83FD21D7 /* 131.253.33.215/32 */, 32, NDPI_PROTOCOL_MICROSOFT_365 }, { 0x84F50000 /* 132.245.0.0/16 */, 16, NDPI_PROTOCOL_MICROSOFT_365 }, { 0x96AB2000 /* 150.171.32.0/22 */, 22, NDPI_PROTOCOL_MICROSOFT_365 }, @@ -8673,6 +8673,8 @@ static ndpi_protocol_match host_match[] = { ".cloudfront.net", "Amazon", NDPI_PROTOCOL_AMAZON, NDPI_PROTOCOL_CATEGORY_WEB, NDPI_PROTOCOL_ACCEPTABLE }, { ".us-west-2.compute.amazonaws.com","Amazon", NDPI_PROTOCOL_AMAZON, NDPI_PROTOCOL_CATEGORY_WEB, NDPI_PROTOCOL_ACCEPTABLE }, + { ".teamviewer.com", "Teamviewer", NDPI_PROTOCOL_TEAMVIEWER, NDPI_PROTOCOL_CATEGORY_REMOTE_ACCESS, NDPI_PROTOCOL_ACCEPTABLE }, + /* Microsoft + Azure */ { ".azure.com", "Microsoft", NDPI_PROTOCOL_MICROSOFT, NDPI_PROTOCOL_CATEGORY_CLOUD, NDPI_PROTOCOL_SAFE }, { ".windows.net", "Microsoft", NDPI_PROTOCOL_MICROSOFT, NDPI_PROTOCOL_CATEGORY_CLOUD, NDPI_PROTOCOL_SAFE }, @@ -8768,6 +8770,10 @@ static ndpi_protocol_match host_match[] = /* http://check.googlezip.net/connect [check browser connectivity] */ // { ".googlezip.net", "Google", NDPI_PROTOCOL_GOOGLE, NDPI_PROTOCOL_CATEGORY_WEB, NDPI_PROTOCOL_SAFE }, + /* + https://github.com/bambenek/block-doh/blob/master/db.doh-redirect + https://github.com/curl/curl/wiki/DNS-over-HTTPS + */ { "dns.google", "DoH_DoT", NDPI_PROTOCOL_DOH_DOT, NDPI_PROTOCOL_CATEGORY_NETWORK, NDPI_PROTOCOL_ACCEPTABLE }, { "mozilla.cloudflare-dns.com", "DoH_DoT", NDPI_PROTOCOL_DOH_DOT, NDPI_PROTOCOL_CATEGORY_NETWORK, NDPI_PROTOCOL_ACCEPTABLE }, /* Firefox */ { "cloudflare-dns.com", "DoH_DoT", NDPI_PROTOCOL_DOH_DOT, NDPI_PROTOCOL_CATEGORY_NETWORK, NDPI_PROTOCOL_ACCEPTABLE }, @@ -8932,6 +8938,7 @@ static ndpi_protocol_match host_match[] = { ".spotify.", "Spotify", NDPI_PROTOCOL_SPOTIFY, NDPI_PROTOCOL_CATEGORY_MUSIC, NDPI_PROTOCOL_FUN }, { "audio-fa.scdn.co", "Spotify", NDPI_PROTOCOL_SPOTIFY, NDPI_PROTOCOL_CATEGORY_MUSIC, NDPI_PROTOCOL_FUN }, + { "spotifycdn.net", "Spotify", NDPI_PROTOCOL_SPOTIFY, NDPI_PROTOCOL_CATEGORY_MUSIC, NDPI_PROTOCOL_FUN }, { "edge-mqtt.facebook.com", "Messenger", NDPI_PROTOCOL_MESSENGER, NDPI_PROTOCOL_CATEGORY_CHAT, NDPI_PROTOCOL_ACCEPTABLE }, { "mqtt-mini.facebook.com", "Messenger", NDPI_PROTOCOL_MESSENGER, NDPI_PROTOCOL_CATEGORY_CHAT, NDPI_PROTOCOL_ACCEPTABLE }, /* Messenger Lite */ diff --git a/src/lib/ndpi_main.c b/src/lib/ndpi_main.c index ca557ea31..fed3c9831 100644 --- a/src/lib/ndpi_main.c +++ b/src/lib/ndpi_main.c @@ -1626,7 +1626,7 @@ u_int16_t ndpi_network_port_ptree_match(struct ndpi_detection_module_struct *ndp || (node->value.uv.additional_user_value == port)) return(node->value.uv.user_value); } - + return(NDPI_PROTOCOL_UNKNOWN); } @@ -2198,10 +2198,10 @@ int ndpi_match_custom_category(struct ndpi_detection_module_struct *ndpi_str, char *name, u_int name_len, ndpi_protocol_category_t *category) { ndpi_protocol_breed_t breed; - u_int16_t id; + u_int16_t id; int rc = ndpi_match_string_protocol_id(ndpi_str->custom_categories.hostnames.ac_automa, name, name_len, &id, category, &breed); - + return(rc); } @@ -2277,6 +2277,9 @@ void ndpi_exit_detection_module(struct ndpi_detection_module_struct *ndpi_str) { if(ndpi_str->stun_cache) ndpi_lru_free_cache(ndpi_str->stun_cache); + if(ndpi_str->msteams_cache) + ndpi_lru_free_cache(ndpi_str->msteams_cache); + if(ndpi_str->protocols_ptree) ndpi_Destroy_Patricia((patricia_tree_t *) ndpi_str->protocols_ptree, free_ptree_data); @@ -2315,7 +2318,7 @@ void ndpi_exit_detection_module(struct ndpi_detection_module_struct *ndpi_str) { #ifdef CUSTOM_NDPI_PROTOCOLS #include "../../../nDPI-custom/ndpi_exit_detection_module.c" #endif - + ndpi_free(ndpi_str); } } @@ -3491,12 +3494,17 @@ static u_int8_t ndpi_detection_get_l4_internal(struct ndpi_detection_module_stru return(0); } +/* ************************************************ */ + void ndpi_apply_flow_protocol_to_packet(struct ndpi_flow_struct *flow, struct ndpi_packet_struct *packet) { memcpy(&packet->detected_protocol_stack, &flow->detected_protocol_stack, sizeof(packet->detected_protocol_stack)); memcpy(&packet->protocol_stack_info, &flow->protocol_stack_info, sizeof(packet->protocol_stack_info)); } -static int ndpi_init_packet_header(struct ndpi_detection_module_struct *ndpi_str, struct ndpi_flow_struct *flow, +/* ************************************************ */ + +static int ndpi_init_packet_header(struct ndpi_detection_module_struct *ndpi_str, + struct ndpi_flow_struct *flow, unsigned short packetlen) { const struct ndpi_iphdr *decaps_iph = NULL; u_int16_t l3len; @@ -3634,7 +3642,10 @@ static int ndpi_init_packet_header(struct ndpi_detection_module_struct *ndpi_str return(0); } -void ndpi_connection_tracking(struct ndpi_detection_module_struct *ndpi_str, struct ndpi_flow_struct *flow) { +/* ************************************************ */ + +void ndpi_connection_tracking(struct ndpi_detection_module_struct *ndpi_str, + struct ndpi_flow_struct *flow) { if(!flow) { return; } else { @@ -3756,10 +3767,12 @@ void ndpi_connection_tracking(struct ndpi_detection_module_struct *ndpi_str, str } } +/* ************************************************ */ + void check_ndpi_other_flow_func(struct ndpi_detection_module_struct *ndpi_str, struct ndpi_flow_struct *flow, NDPI_SELECTION_BITMASK_PROTOCOL_SIZE *ndpi_selection_packet) { if(!flow) - return; + return; void *func = NULL; u_int32_t a; @@ -3797,6 +3810,8 @@ void check_ndpi_other_flow_func(struct ndpi_detection_module_struct *ndpi_str, s } } +/* ************************************************ */ + void check_ndpi_udp_flow_func(struct ndpi_detection_module_struct *ndpi_str, struct ndpi_flow_struct *flow, NDPI_SELECTION_BITMASK_PROTOCOL_SIZE *ndpi_selection_packet) { void *func = NULL; @@ -3838,7 +3853,10 @@ void check_ndpi_udp_flow_func(struct ndpi_detection_module_struct *ndpi_str, str } } -void check_ndpi_tcp_flow_func(struct ndpi_detection_module_struct *ndpi_str, struct ndpi_flow_struct *flow, +/* ************************************************ */ + +void check_ndpi_tcp_flow_func(struct ndpi_detection_module_struct *ndpi_str, + struct ndpi_flow_struct *flow, NDPI_SELECTION_BITMASK_PROTOCOL_SIZE *ndpi_selection_packet) { void *func = NULL; u_int32_t a; @@ -3931,16 +3949,16 @@ u_int16_t ndpi_guess_host_protocol_id(struct ndpi_detection_module_struct *ndpi_ if(flow->packet.iph) { struct in_addr addr; u_int16_t sport, dport; - + addr.s_addr = flow->packet.iph->saddr; - + if((flow->l4_proto == IPPROTO_TCP) && flow->packet.tcp) sport = flow->packet.tcp->source, dport = flow->packet.tcp->dest; else if((flow->l4_proto == IPPROTO_UDP) && flow->packet.udp) sport = flow->packet.udp->source, dport = flow->packet.udp->dest; else sport = dport = 0; - + /* guess host protocol */ ret = ndpi_network_port_ptree_match(ndpi_str, &addr, sport); @@ -4342,11 +4360,79 @@ static void ndpi_reset_packet_line_info(struct ndpi_packet_struct *packet) { /* ********************************************************************************* */ -#if 0 -static u_int16_t ndpi_checK_flow_port(, u_int16_t sport, u_int16_t dport) { +static int ndpi_check_protocol_port_mismatch_exceptions(struct ndpi_detection_module_struct *ndpi_str, + struct ndpi_flow_struct *flow, + ndpi_default_ports_tree_node_t *expected_proto, + ndpi_protocol *returned_proto) { + /* + For TLS (and other protocols) it is not simple to guess the exact protocol so before + triggering an alert we need to make sure what we have exhausted all the possible + options available + */ + if(returned_proto->master_protocol == NDPI_PROTOCOL_TLS) { + switch(expected_proto->proto->protoId) { + case NDPI_PROTOCOL_MAIL_IMAPS: + case NDPI_PROTOCOL_MAIL_POPS: + case NDPI_PROTOCOL_MAIL_SMTPS: + return(1); /* This is a reasonable exception */ + break; + } + } + + return(0); +} + +/* ********************************************************************************* */ + +static void ndpi_reconcile_protocols(struct ndpi_detection_module_struct *ndpi_str, + struct ndpi_flow_struct *flow, + ndpi_protocol *ret) { + /* + Skype for a host doing MS Teams means MS Teams + (MS Teams uses Skype as transport protocol for voice/video) + */ + + switch(ret->app_protocol) { + case NDPI_PROTOCOL_MSTEAMS: + if(flow->packet.iph && flow->packet.tcp) { + // printf("====>> NDPI_PROTOCOL_MSTEAMS\n"); + + if(ndpi_str->msteams_cache == NULL) + ndpi_str->msteams_cache = ndpi_lru_cache_init(1024); + + if(ndpi_str->msteams_cache) + ndpi_lru_add_to_cache(ndpi_str->msteams_cache, + flow->packet.iph->saddr, + flow->packet.tick_timestamp & 0xFFFF /* 16 bit */); + } + break; + + case NDPI_PROTOCOL_SKYPE: + case NDPI_PROTOCOL_SKYPE_CALL: + if(flow->packet.iph + && flow->packet.udp + && ndpi_str->msteams_cache) { + u_int16_t when; + + if(ndpi_lru_find_cache(ndpi_str->msteams_cache, flow->packet.iph->saddr, + &when, 0 /* Don't remove it as it can be used for other connections */)) { + u_int16_t tdiff = (flow->packet.tick_timestamp & 0xFFFF) - when; + + if(tdiff < 60 /* sec */) { + // printf("====>> NDPI_PROTOCOL_SKYPE(_CALL) -> NDPI_PROTOCOL_MSTEAMS [%u]\n", tdiff); + ret->app_protocol = NDPI_PROTOCOL_MSTEAMS; + + /* Refresh cache */ + ndpi_lru_add_to_cache(ndpi_str->msteams_cache, + flow->packet.iph->saddr, + flow->packet.tick_timestamp & 0xFFFF /* 16 bit */); + } + } + } + break; + } /* switch */ } -#endif /* ********************************************************************************* */ @@ -4586,7 +4672,7 @@ ndpi_protocol ndpi_detection_process_packet(struct ndpi_detection_module_struct if((!flow->risk_checked) && (ret.master_protocol != NDPI_PROTOCOL_UNKNOWN)) { ndpi_default_ports_tree_node_t *found; u_int16_t *default_ports, sport, dport; - + if(flow->packet.udp) found = ndpi_get_guessed_protocol_id(ndpi_str, IPPROTO_UDP, sport = ntohs(flow->packet.udp->source), @@ -4596,7 +4682,7 @@ ndpi_protocol ndpi_detection_process_packet(struct ndpi_detection_module_struct found = ndpi_get_guessed_protocol_id(ndpi_str, IPPROTO_TCP, sport = ntohs(flow->packet.tcp->source), dport = ntohs(flow->packet.tcp->dest)), - default_ports = ndpi_str->proto_defaults[ret.master_protocol].tcp_default_ports; + default_ports = ndpi_str->proto_defaults[ret.master_protocol].tcp_default_ports; else found = NULL, default_ports = NULL; @@ -4604,10 +4690,12 @@ ndpi_protocol ndpi_detection_process_packet(struct ndpi_detection_module_struct && (found->proto->protoId != NDPI_PROTOCOL_UNKNOWN) && (found->proto->protoId != ret.master_protocol)) { // printf("******** %u / %u\n", found->proto->protoId, ret.master_protocol); - NDPI_SET_BIT(flow->risk, NDPI_KNOWN_PROTOCOL_ON_NON_STANDARD_PORT); + + if(!ndpi_check_protocol_port_mismatch_exceptions(ndpi_str, flow, found, &ret)) + NDPI_SET_BIT(flow->risk, NDPI_KNOWN_PROTOCOL_ON_NON_STANDARD_PORT); } else if(default_ports && (default_ports[0] != 0)) { u_int8_t found = 0, i; - + for(i=0; (i<MAX_DEFAULT_PORTS) && (default_ports[i] != 0); i++) { if((default_ports[i] == sport) || (default_ports[i] == dport)) { found = 1; @@ -4620,10 +4708,11 @@ ndpi_protocol ndpi_detection_process_packet(struct ndpi_detection_module_struct NDPI_SET_BIT(flow->risk, NDPI_KNOWN_PROTOCOL_ON_NON_STANDARD_PORT); } } - + flow->risk_checked = 1; } - + + ndpi_reconcile_protocols(ndpi_str, flow, &ret); invalidate_ptr: /* @@ -4749,29 +4838,34 @@ u_int32_t ndpi_bytestream_to_ipv4(const u_int8_t *str, u_int16_t max_chars_to_re u_int16_t read = 0; u_int16_t oldread; u_int32_t c; + /* ip address must be X.X.X.X with each X between 0 and 255 */ oldread = read; c = ndpi_bytestream_to_number(str, max_chars_to_read, &read); if(c > 255 || oldread == read || max_chars_to_read == read || str[read] != '.') return(0); + read++; val = c << 24; oldread = read; c = ndpi_bytestream_to_number(&str[read], max_chars_to_read - read, &read); if(c > 255 || oldread == read || max_chars_to_read == read || str[read] != '.') return(0); + read++; val = val + (c << 16); oldread = read; c = ndpi_bytestream_to_number(&str[read], max_chars_to_read - read, &read); if(c > 255 || oldread == read || max_chars_to_read == read || str[read] != '.') return(0); + read++; val = val + (c << 8); oldread = read; c = ndpi_bytestream_to_number(&str[read], max_chars_to_read - read, &read); if(c > 255 || oldread == read || max_chars_to_read == read) return(0); + val = val + c; *bytes_read = *bytes_read + read; @@ -4805,14 +4899,16 @@ void ndpi_parse_packet_line_info(struct ndpi_detection_module_struct *ndpi_str, if(get_u_int16_t(packet->payload, a) == ntohs(0x0d0a)) { /* If end of line char sequence CR+NL "\r\n", process line */ - if(get_u_int16_t(packet->payload, a+2) == ntohs(0x0d0a)) { + if(((a + 3) <= packet->payload_packet_len) + && (get_u_int16_t(packet->payload, a+2) == ntohs(0x0d0a))) { /* \r\n\r\n */ int diff; /* No unsigned ! */ u_int32_t a1 = a + 4; - diff = ndpi_min(packet->payload_packet_len-a1, sizeof(flow->initial_binary_bytes)); - + diff = packet->payload_packet_len - a1; + if(diff > 0) { + diff = ndpi_min(diff, sizeof(flow->initial_binary_bytes)); memcpy(&flow->initial_binary_bytes, &packet->payload[a1], diff); flow->initial_binary_bytes_len = diff; } @@ -5814,27 +5910,21 @@ char *ndpi_strnstr(const char *s, const char *find, size_t slen) { /* * Same as ndpi_strnstr but case-insensitive */ -char *ndpi_strncasestr(const char *s, const char *find, size_t slen) { - char c; - size_t len; - - if((c = *find++) != '\0') { - len = strlen(find); - do { - char sc; - - do { - if(slen-- < 1 || (sc = *s++) == '\0') - return(NULL); - } while (sc != c); - - if(len > slen) - return(NULL); - } while (strncasecmp(s, find, len) != 0); - - s--; +const char * ndpi_strncasestr(const char *str1, const char *str2, size_t len) { + size_t str1_len = strnlen(str1, len); + size_t str2_len = strlen(str2); + size_t i; + + for(i = 0; i < (str1_len - str2_len + 1); i++){ + if(str1[0] == '\0') + return NULL; + else if(strncasecmp(str1, str2, str2_len) == 0) + return(str1); + + str1++; } - return((char *) s); + + return NULL; } /* ****************************************************** */ @@ -6065,6 +6155,9 @@ void ndpi_free_flow(struct ndpi_flow_struct *flow) { if(flow->l4.tcp.tls.srv_cert_fingerprint_ctx) ndpi_free(flow->l4.tcp.tls.srv_cert_fingerprint_ctx); + + if(flow->protos.stun_ssl.ssl.encrypted_sni.esni) + ndpi_free(flow->protos.stun_ssl.ssl.encrypted_sni.esni); } if(flow->l4_proto == IPPROTO_TCP) { diff --git a/src/lib/ndpi_serializer.c b/src/lib/ndpi_serializer.c index d8a0d13fc..a50f98860 100644 --- a/src/lib/ndpi_serializer.c +++ b/src/lib/ndpi_serializer.c @@ -146,15 +146,27 @@ void ndpi_reset_serializer(ndpi_serializer *_serializer) { if(serializer->fmt == ndpi_serialization_format_json) { u_int32_t buff_diff; - serializer->status.size_used = 0; - buff_diff = serializer->buffer_size - serializer->status.size_used; + serializer->status.buffer.size_used = 0; + buff_diff = serializer->buffer.size - serializer->status.buffer.size_used; /* Note: please keep a space at the beginning as it is used for arrays when an end-of-record is used */ - serializer->status.size_used += snprintf((char *) &serializer->buffer[serializer->status.size_used], buff_diff, " {}"); - } else if(serializer->fmt == ndpi_serialization_format_csv) - serializer->status.size_used = 0; - else /* ndpi_serialization_format_tlv */ - serializer->status.size_used = 2 * sizeof(u_int8_t); + serializer->status.buffer.size_used += snprintf((char *) &serializer->buffer.data[serializer->status.buffer.size_used], buff_diff, " {}"); + } else if(serializer->fmt == ndpi_serialization_format_csv) { + serializer->status.header.size_used = 0; + serializer->status.buffer.size_used = 0; + } else { /* ndpi_serialization_format_tlv */ + serializer->status.buffer.size_used = 2 * sizeof(u_int8_t); + } +} + +/* ********************************** */ + +static int ndpi_init_serializer_buffer(ndpi_private_serializer_buffer *buffer, u_int32_t buffer_size) { + buffer->initial_size = buffer->size = buffer_size; + buffer->data = (u_int8_t *) calloc(buffer->size, sizeof(u_int8_t)); + if(buffer->data == NULL) + return -1; + return 0; } /* ********************************** */ @@ -166,16 +178,22 @@ int ndpi_init_serializer_ll(ndpi_serializer *_serializer, memset(serializer, 0, sizeof(ndpi_private_serializer)); - serializer->initial_buffer_size = serializer->buffer_size = buffer_size; - serializer->buffer = (u_int8_t *) calloc(serializer->buffer_size, sizeof(u_int8_t)); + serializer->fmt = fmt; - if(serializer->buffer == NULL) + if (ndpi_init_serializer_buffer(&serializer->buffer, buffer_size) != 0) return(-1); + + if(serializer->fmt == ndpi_serialization_format_json) { + /* nothing to do */ - serializer->fmt = fmt; + } else if (fmt == ndpi_serialization_format_csv) { + if (ndpi_init_serializer_buffer(&serializer->header, NDPI_SERIALIZER_DEFAULT_HEADER_SIZE) != 0) + return(-1); - serializer->buffer[0] = 1; /* version */ - serializer->buffer[1] = (u_int8_t) fmt; + } else /* ndpi_serialization_format_tlv */ { + serializer->buffer.data[0] = 1; /* version */ + serializer->buffer.data[1] = (u_int8_t) fmt; + } serializer->csv_separator[0] = ','; serializer->csv_separator[1] = '\0'; @@ -194,45 +212,106 @@ int ndpi_init_serializer(ndpi_serializer *_serializer, /* ********************************** */ -static inline int ndpi_extend_serializer_buffer(ndpi_serializer *_serializer, u_int32_t min_len) { +static inline int ndpi_extend_serializer_buffer(ndpi_private_serializer_buffer *buffer, u_int32_t min_len) { u_int32_t new_size; void *r; - ndpi_private_serializer *serializer = (ndpi_private_serializer*)_serializer; if(min_len < NDPI_SERIALIZER_DEFAULT_BUFFER_INCR) { - if(serializer->initial_buffer_size < NDPI_SERIALIZER_DEFAULT_BUFFER_INCR) { - if(min_len < serializer->initial_buffer_size) - min_len = serializer->initial_buffer_size; + if(buffer->initial_size < NDPI_SERIALIZER_DEFAULT_BUFFER_INCR) { + if(min_len < buffer->initial_size) + min_len = buffer->initial_size; } else { min_len = NDPI_SERIALIZER_DEFAULT_BUFFER_INCR; } } - new_size = serializer->buffer_size + min_len; + new_size = buffer->size + min_len; new_size = ((new_size / 4) + 1) * 4; /* required by zmq encryption */ - r = realloc((void *) serializer->buffer, new_size); + r = realloc((void *) buffer->data, new_size); if(r == NULL) return(-1); - serializer->buffer = r; - serializer->buffer_size = new_size; + buffer->data = r; + buffer->size = new_size; return(0); } /* ********************************** */ +static inline int ndpi_serializer_check_header_room(ndpi_private_serializer *serializer, u_int32_t needed) { + u_int32_t buff_diff = serializer->header.size - serializer->status.header.size_used; + + if (buff_diff < needed) + if (ndpi_extend_serializer_buffer(&serializer->header, needed - buff_diff) < 0) + return -1; + + buff_diff = serializer->header.size - serializer->status.header.size_used; + + return buff_diff; +} + +/* ********************************** */ + +static inline int ndpi_serializer_header_uint32(ndpi_private_serializer *serializer, u_int32_t key) { + int room; + + if (serializer->status.flags & NDPI_SERIALIZER_STATUS_HDR_DONE) + return 0; + + room = ndpi_serializer_check_header_room(serializer, 12); + + if (room < 0) + return -1; + + serializer->status.header.size_used += snprintf((char *) &serializer->header.data[serializer->status.header.size_used], + room, "%s%u", (serializer->status.header.size_used > 0) ? serializer->csv_separator : "", key); + + return 0; +} + +/* ********************************** */ + +static inline int ndpi_serializer_header_string(ndpi_private_serializer *serializer, const char *key, u_int16_t klen) { + int room; + + if (serializer->status.flags & NDPI_SERIALIZER_STATUS_HDR_DONE) + return 0; + + room = ndpi_serializer_check_header_room(serializer, klen + 4); + + if (room < 0) + return -1; + + if (serializer->status.header.size_used > 0) { + int slen = strlen(serializer->csv_separator); + memcpy(&serializer->header.data[serializer->status.header.size_used], serializer->csv_separator, slen); + serializer->status.header.size_used += slen; + } + + if (klen > 0) { + memcpy(&serializer->header.data[serializer->status.header.size_used], key, klen); + serializer->status.header.size_used += klen; + } + + serializer->header.data[serializer->status.header.size_used] = '\0'; + + return 0; +} + +/* ********************************** */ + char* ndpi_serializer_get_buffer(ndpi_serializer *_serializer, u_int32_t *buffer_len) { ndpi_private_serializer *serializer = (ndpi_private_serializer*)_serializer; - char *buf = (char*)serializer->buffer; + char *buf = (char*)serializer->buffer.data; - /* NULL terminate the buffer if there is space available */ - if(serializer->buffer_size > serializer->status.size_used) - serializer->buffer[serializer->status.size_used] = '\0'; + /* NULL terminate the buffer */ + if(serializer->buffer.size > serializer->status.buffer.size_used) /* safety check */ + serializer->buffer.data[serializer->status.buffer.size_used] = '\0'; - *buffer_len = serializer->status.size_used; + *buffer_len = serializer->status.buffer.size_used; if(serializer->fmt == ndpi_serialization_format_json) { while((buf[0] == '\0') || (buf[0] == ' ')) @@ -245,13 +324,13 @@ char* ndpi_serializer_get_buffer(ndpi_serializer *_serializer, u_int32_t *buffer /* ********************************** */ u_int32_t ndpi_serializer_get_buffer_len(ndpi_serializer *_serializer) { - return(((ndpi_private_serializer*)_serializer)->status.size_used); + return(((ndpi_private_serializer*)_serializer)->status.buffer.size_used); } /* ********************************** */ u_int32_t ndpi_serializer_get_internal_buffer_size(ndpi_serializer *_serializer) { - return(((ndpi_private_serializer*)_serializer)->buffer_size); + return(((ndpi_private_serializer*)_serializer)->buffer.size); } /* ********************************** */ @@ -260,10 +339,10 @@ int ndpi_serializer_set_buffer_len(ndpi_serializer *_serializer, u_int32_t l) { ndpi_private_serializer *p = (ndpi_private_serializer*)_serializer; if(p) { - if(p->buffer_size <= l) + if(p->buffer.size <= l) return(-1); /* Invalid size */ - p->status.size_used = l; + p->status.buffer.size_used = l; return(0); } @@ -272,6 +351,27 @@ int ndpi_serializer_set_buffer_len(ndpi_serializer *_serializer, u_int32_t l) { /* ********************************** */ +/* Return the header automatically built from keys (CSV only) */ +char* ndpi_serializer_get_header(ndpi_serializer *_serializer, u_int32_t *buffer_len) { + ndpi_private_serializer *serializer = (ndpi_private_serializer*)_serializer; + char *buf = (char*)serializer->header.data; + + if(buf == NULL) { + *buffer_len = 0; + return ""; + } + + /* NULL terminate the buffer */ + if(serializer->header.size > serializer->status.header.size_used) /* safety check */ + serializer->header.data[serializer->status.header.size_used] = '\0'; + + *buffer_len = serializer->status.header.size_used; + + return(buf); +} + +/* ********************************** */ + void ndpi_serializer_set_csv_separator(ndpi_serializer *_serializer, char separator) { ndpi_private_serializer *serializer = (ndpi_private_serializer*)_serializer; @@ -283,10 +383,16 @@ void ndpi_serializer_set_csv_separator(ndpi_serializer *_serializer, char separa void ndpi_term_serializer(ndpi_serializer *_serializer) { ndpi_private_serializer *serializer = (ndpi_private_serializer*)_serializer; - if(serializer->buffer) { - free(serializer->buffer); - serializer->buffer_size = 0; - serializer->buffer = NULL; + if(serializer->buffer.data) { + free(serializer->buffer.data); + serializer->buffer.size = 0; + serializer->buffer.data = NULL; + } + + if(serializer->header.data) { + free(serializer->header.data); + serializer->header.size = 0; + serializer->header.data = NULL; } } @@ -296,8 +402,8 @@ static inline void ndpi_serialize_single_uint8(ndpi_private_serializer *serializ u_int8_t s) { u_int8_t v = s; - memcpy(&serializer->buffer[serializer->status.size_used], &v, sizeof(u_int8_t)); - serializer->status.size_used += sizeof(u_int8_t); + memcpy(&serializer->buffer.data[serializer->status.buffer.size_used], &v, sizeof(u_int8_t)); + serializer->status.buffer.size_used += sizeof(u_int8_t); } /* ********************************** */ @@ -306,8 +412,8 @@ static inline void ndpi_serialize_single_uint16(ndpi_private_serializer *seriali u_int16_t s) { u_int16_t v = htons(s); - memcpy(&serializer->buffer[serializer->status.size_used], &v, sizeof(u_int16_t)); - serializer->status.size_used += sizeof(u_int16_t); + memcpy(&serializer->buffer.data[serializer->status.buffer.size_used], &v, sizeof(u_int16_t)); + serializer->status.buffer.size_used += sizeof(u_int16_t); } /* ********************************** */ @@ -316,8 +422,8 @@ static inline void ndpi_serialize_single_uint32(ndpi_private_serializer *seriali u_int32_t s) { u_int32_t v = htonl(s); - memcpy(&serializer->buffer[serializer->status.size_used], &v, sizeof(u_int32_t)); - serializer->status.size_used += sizeof(u_int32_t); + memcpy(&serializer->buffer.data[serializer->status.buffer.size_used], &v, sizeof(u_int32_t)); + serializer->status.buffer.size_used += sizeof(u_int32_t); } /* ********************************** */ @@ -326,8 +432,8 @@ static inline void ndpi_serialize_single_uint64(ndpi_private_serializer *seriali u_int64_t s) { u_int64_t v = ndpi_htonll(s); - memcpy(&serializer->buffer[serializer->status.size_used], &v, sizeof(u_int64_t)); - serializer->status.size_used += sizeof(u_int64_t); + memcpy(&serializer->buffer.data[serializer->status.buffer.size_used], &v, sizeof(u_int64_t)); + serializer->status.buffer.size_used += sizeof(u_int64_t); } /* ********************************** */ @@ -335,8 +441,8 @@ static inline void ndpi_serialize_single_uint64(ndpi_private_serializer *seriali /* TODO: fix portability across platforms */ static inline void ndpi_serialize_single_float(ndpi_private_serializer *serializer, float s) { - memcpy(&serializer->buffer[serializer->status.size_used], &s, sizeof(s)); - serializer->status.size_used += sizeof(float); + memcpy(&serializer->buffer.data[serializer->status.buffer.size_used], &s, sizeof(s)); + serializer->status.buffer.size_used += sizeof(float); } /* ********************************** */ @@ -345,69 +451,69 @@ static inline void ndpi_serialize_single_string(ndpi_private_serializer *seriali const char *s, u_int16_t slen) { u_int16_t l = htons(slen); - memcpy(&serializer->buffer[serializer->status.size_used], &l, sizeof(u_int16_t)); - serializer->status.size_used += sizeof(u_int16_t); + memcpy(&serializer->buffer.data[serializer->status.buffer.size_used], &l, sizeof(u_int16_t)); + serializer->status.buffer.size_used += sizeof(u_int16_t); if(slen > 0) - memcpy(&serializer->buffer[serializer->status.size_used], s, slen); + memcpy(&serializer->buffer.data[serializer->status.buffer.size_used], s, slen); - serializer->status.size_used += slen; + serializer->status.buffer.size_used += slen; } /* ********************************** */ static inline void ndpi_deserialize_single_uint8(ndpi_private_deserializer *deserializer, u_int32_t offset, u_int8_t *s) { - *s = (*((u_int8_t *) &deserializer->buffer[offset])); + *s = (*((u_int8_t *) &deserializer->buffer.data[offset])); } /* ********************************** */ static inline void ndpi_deserialize_single_uint16(ndpi_private_deserializer *deserializer, u_int32_t offset, u_int16_t *s) { - *s = ntohs(*((u_int16_t *) &deserializer->buffer[offset])); + *s = ntohs(*((u_int16_t *) &deserializer->buffer.data[offset])); } /* ********************************** */ static inline void ndpi_deserialize_single_uint32(ndpi_private_deserializer *deserializer, u_int32_t offset, u_int32_t *s) { - *s = ntohl(*((u_int32_t *) &deserializer->buffer[offset])); + *s = ntohl(*((u_int32_t *) &deserializer->buffer.data[offset])); } /* ********************************** */ static inline void ndpi_deserialize_single_int8(ndpi_private_deserializer *deserializer, u_int32_t offset, int8_t *s) { - *s = (*((int8_t *) &deserializer->buffer[offset])); + *s = (*((int8_t *) &deserializer->buffer.data[offset])); } /* ********************************** */ static inline void ndpi_deserialize_single_int16(ndpi_private_deserializer *deserializer, u_int32_t offset, int16_t *s) { - *s = ntohs(*((int16_t *) &deserializer->buffer[offset])); + *s = ntohs(*((int16_t *) &deserializer->buffer.data[offset])); } /* ********************************** */ static inline void ndpi_deserialize_single_int32(ndpi_private_deserializer *deserializer, u_int32_t offset, int32_t *s) { - *s = ntohl(*((int32_t *) &deserializer->buffer[offset])); + *s = ntohl(*((int32_t *) &deserializer->buffer.data[offset])); } /* ********************************** */ static inline void ndpi_deserialize_single_uint64(ndpi_private_deserializer *deserializer, u_int32_t offset, u_int64_t *s) { - *s = ndpi_ntohll(*(u_int64_t*)&deserializer->buffer[offset]); + *s = ndpi_ntohll(*(u_int64_t*)&deserializer->buffer.data[offset]); } /* ********************************** */ static inline void ndpi_deserialize_single_int64(ndpi_private_deserializer *deserializer, u_int32_t offset, int64_t *s) { - *s = ndpi_ntohll(*(int64_t*)&deserializer->buffer[offset]); + *s = ndpi_ntohll(*(int64_t*)&deserializer->buffer.data[offset]); } /* ********************************** */ @@ -415,15 +521,15 @@ static inline void ndpi_deserialize_single_int64(ndpi_private_deserializer *dese /* TODO: fix portability across platforms */ static inline void ndpi_deserialize_single_float(ndpi_private_deserializer *deserializer, u_int32_t offset, float *s) { - *s = *(float*)&deserializer->buffer[offset]; + *s = *(float*)&deserializer->buffer.data[offset]; } /* ********************************** */ static inline void ndpi_deserialize_single_string(ndpi_private_deserializer *deserializer, u_int32_t offset, ndpi_string *v) { - v->str_len = ntohs(*((u_int16_t *) &deserializer->buffer[offset])); - v->str = (char *) &deserializer->buffer[offset + sizeof(u_int16_t)]; + v->str_len = ntohs(*((u_int16_t *) &deserializer->buffer.data[offset])); + v->str = (char *) &deserializer->buffer.data[offset + sizeof(u_int16_t)]; } /* ********************************** */ @@ -436,39 +542,39 @@ static inline void ndpi_deserialize_single_string(ndpi_private_deserializer *des int ndpi_serialize_raw_record(ndpi_serializer *_serializer, u_char *record, u_int32_t record_len) { ndpi_private_serializer *serializer = (ndpi_private_serializer*)_serializer; - u_int32_t buff_diff = serializer->buffer_size - serializer->status.size_used; + u_int32_t buff_diff = serializer->buffer.size - serializer->status.buffer.size_used; u_int16_t needed = record_len; u_int8_t add_comma = 0; if(serializer->fmt == ndpi_serialization_format_json) { needed += 1; - if(serializer->status.size_used == 3) /* Empty buffer [{} */ - serializer->status.size_used = 2; /* Remove {} */ + if(serializer->status.buffer.size_used == 3) /* Empty buffer [{} */ + serializer->status.buffer.size_used = 2; /* Remove {} */ else needed += 2, add_comma = 1; } if(buff_diff < needed) { - if(ndpi_extend_serializer_buffer(_serializer, needed - buff_diff) < 0) + if(ndpi_extend_serializer_buffer(&serializer->buffer, needed - buff_diff) < 0) return(-1); - buff_diff = serializer->buffer_size - serializer->status.size_used; + buff_diff = serializer->buffer.size - serializer->status.buffer.size_used; } if(serializer->fmt == ndpi_serialization_format_json) { if(add_comma) - serializer->buffer[serializer->status.size_used-1] = ','; + serializer->buffer.data[serializer->status.buffer.size_used-1] = ','; else - serializer->status.size_used--; + serializer->status.buffer.size_used--; } - memcpy(&serializer->buffer[serializer->status.size_used], record, record_len); - serializer->status.size_used += record_len; + memcpy(&serializer->buffer.data[serializer->status.buffer.size_used], record, record_len); + serializer->status.buffer.size_used += record_len; if(serializer->fmt == ndpi_serialization_format_json) { - serializer->buffer[serializer->status.size_used] = ']'; - if(add_comma) serializer->status.size_used++; + serializer->buffer.data[serializer->status.buffer.size_used] = ']'; + if(add_comma) serializer->status.buffer.size_used++; } ndpi_serialize_end_of_record(_serializer); @@ -480,29 +586,31 @@ int ndpi_serialize_raw_record(ndpi_serializer *_serializer, int ndpi_serialize_end_of_record(ndpi_serializer *_serializer) { ndpi_private_serializer *serializer = (ndpi_private_serializer*)_serializer; - u_int32_t buff_diff = serializer->buffer_size - serializer->status.size_used; + u_int32_t buff_diff = serializer->buffer.size - serializer->status.buffer.size_used; u_int16_t needed = sizeof(u_int8_t) /* type */; if(serializer->fmt == ndpi_serialization_format_json) needed += 1; if(buff_diff < needed) { - if(ndpi_extend_serializer_buffer(_serializer, needed - buff_diff) < 0) + if(ndpi_extend_serializer_buffer(&serializer->buffer, needed - buff_diff) < 0) return(-1); - buff_diff = serializer->buffer_size - serializer->status.size_used; + buff_diff = serializer->buffer.size - serializer->status.buffer.size_used; } - if(serializer->fmt == ndpi_serialization_format_json) { + if(serializer->fmt == ndpi_serialization_format_csv) { + serializer->status.flags |= NDPI_SERIALIZER_STATUS_HDR_DONE; + } else if(serializer->fmt == ndpi_serialization_format_json) { if(!(serializer->status.flags & NDPI_SERIALIZER_STATUS_ARRAY)) { - serializer->buffer[0] = '['; - serializer->status.size_used += snprintf((char *) &serializer->buffer[serializer->status.size_used], + serializer->buffer.data[0] = '['; + serializer->status.buffer.size_used += snprintf((char *) &serializer->buffer.data[serializer->status.buffer.size_used], buff_diff, "]"); } serializer->status.flags |= NDPI_SERIALIZER_STATUS_ARRAY | NDPI_SERIALIZER_STATUS_EOR; serializer->status.flags &= ~NDPI_SERIALIZER_STATUS_COMMA; } else { - serializer->buffer[serializer->status.size_used++] = ndpi_serialization_end_of_record; + serializer->buffer.data[serializer->status.buffer.size_used++] = ndpi_serialization_end_of_record; } serializer->status.flags &= ~NDPI_SERIALIZER_STATUS_NOT_EMPTY; @@ -516,19 +624,28 @@ static inline void ndpi_serialize_json_pre(ndpi_serializer *_serializer) { ndpi_private_serializer *serializer = (ndpi_private_serializer*)_serializer; if(serializer->status.flags & NDPI_SERIALIZER_STATUS_EOR) { - serializer->status.size_used--; /* Remove ']' */ + serializer->status.buffer.size_used--; /* Remove ']' */ serializer->status.flags &= ~NDPI_SERIALIZER_STATUS_EOR; - serializer->buffer[serializer->status.size_used++] = ','; - serializer->buffer[serializer->status.size_used++] = '{'; + serializer->buffer.data[serializer->status.buffer.size_used++] = ','; + serializer->buffer.data[serializer->status.buffer.size_used++] = '{'; } else { if(serializer->status.flags & NDPI_SERIALIZER_STATUS_ARRAY) - serializer->status.size_used--; /* Remove ']'*/ - serializer->status.size_used--; /* Remove '}'*/ + serializer->status.buffer.size_used--; /* Remove ']' */ - if(serializer->status.flags & NDPI_SERIALIZER_STATUS_SOB) - serializer->status.flags &= ~NDPI_SERIALIZER_STATUS_SOB; - else if(serializer->status.flags & NDPI_SERIALIZER_STATUS_COMMA) - serializer->buffer[serializer->status.size_used++] = ','; + serializer->status.buffer.size_used--; /* Remove '}' */ + + if(serializer->status.flags & NDPI_SERIALIZER_STATUS_LIST) { + serializer->status.buffer.size_used--; /* Remove ']' */ + if(serializer->status.flags & NDPI_SERIALIZER_STATUS_SOL) + serializer->status.flags &= ~NDPI_SERIALIZER_STATUS_SOL; + else + serializer->buffer.data[serializer->status.buffer.size_used++] = ','; + } else { + if(serializer->status.flags & NDPI_SERIALIZER_STATUS_SOB) + serializer->status.flags &= ~NDPI_SERIALIZER_STATUS_SOB; + else if(serializer->status.flags & NDPI_SERIALIZER_STATUS_COMMA) + serializer->buffer.data[serializer->status.buffer.size_used++] = ','; + } } } @@ -537,9 +654,13 @@ static inline void ndpi_serialize_json_pre(ndpi_serializer *_serializer) { static inline void ndpi_serialize_json_post(ndpi_serializer *_serializer) { ndpi_private_serializer *serializer = (ndpi_private_serializer*)_serializer; - serializer->buffer[serializer->status.size_used++] = '}'; + if(serializer->status.flags & NDPI_SERIALIZER_STATUS_LIST) + serializer->buffer.data[serializer->status.buffer.size_used++] = ']'; + + serializer->buffer.data[serializer->status.buffer.size_used++] = '}'; + if(serializer->status.flags & NDPI_SERIALIZER_STATUS_ARRAY) - serializer->buffer[serializer->status.size_used++] = ']'; + serializer->buffer.data[serializer->status.buffer.size_used++] = ']'; serializer->status.flags |= NDPI_SERIALIZER_STATUS_COMMA; } @@ -568,7 +689,7 @@ static inline ndpi_serialization_type ndpi_serialize_key_uint32(ndpi_private_ser int ndpi_serialize_uint32_uint32(ndpi_serializer *_serializer, u_int32_t key, u_int32_t value) { ndpi_private_serializer *serializer = (ndpi_private_serializer*)_serializer; - u_int32_t buff_diff = serializer->buffer_size - serializer->status.size_used; + u_int32_t buff_diff = serializer->buffer.size - serializer->status.buffer.size_used; u_int16_t needed = sizeof(u_int8_t) /* type */ + sizeof(u_int32_t) /* key */ + @@ -578,23 +699,34 @@ int ndpi_serialize_uint32_uint32(ndpi_serializer *_serializer, needed += 24; if(buff_diff < needed) { - if(ndpi_extend_serializer_buffer(_serializer, needed - buff_diff) < 0) + if(ndpi_extend_serializer_buffer(&serializer->buffer, needed - buff_diff) < 0) return(-1); - buff_diff = serializer->buffer_size - serializer->status.size_used; + buff_diff = serializer->buffer.size - serializer->status.buffer.size_used; } if(serializer->fmt == ndpi_serialization_format_json) { ndpi_serialize_json_pre(_serializer); - serializer->status.size_used += snprintf((char *) &serializer->buffer[serializer->status.size_used], buff_diff, - "\"%u\":%u", key, value); + + if (!(serializer->status.flags & NDPI_SERIALIZER_STATUS_LIST)) { + serializer->status.buffer.size_used += snprintf((char *) + &serializer->buffer.data[serializer->status.buffer.size_used], + buff_diff, "\"%u\":", key); + buff_diff = serializer->buffer.size - serializer->status.buffer.size_used; + } + + serializer->status.buffer.size_used += snprintf((char *) + &serializer->buffer.data[serializer->status.buffer.size_used], + buff_diff, "%u", value); + ndpi_serialize_json_post(_serializer); } else if(serializer->fmt == ndpi_serialization_format_csv) { - serializer->status.size_used += snprintf((char *) &serializer->buffer[serializer->status.size_used], buff_diff, - "%s%u", (serializer->status.size_used > 0) ? serializer->csv_separator : "", value); + if (ndpi_serializer_header_uint32(serializer, key) < 0) return(-1); + serializer->status.buffer.size_used += snprintf((char *) &serializer->buffer.data[serializer->status.buffer.size_used], buff_diff, + "%s%u", (serializer->status.buffer.size_used > 0) ? serializer->csv_separator : "", value); } else { ndpi_serialization_type kt; u_int8_t type = 0; - u_int32_t type_offset = serializer->status.size_used++; + u_int32_t type_offset = serializer->status.buffer.size_used++; kt = ndpi_serialize_key_uint32(serializer, key); type = (kt << 4); @@ -610,7 +742,7 @@ int ndpi_serialize_uint32_uint32(ndpi_serializer *_serializer, type |= ndpi_serialization_uint32; } - serializer->buffer[type_offset] = type; + serializer->buffer.data[type_offset] = type; } serializer->status.flags |= NDPI_SERIALIZER_STATUS_NOT_EMPTY; @@ -622,7 +754,7 @@ int ndpi_serialize_uint32_uint32(ndpi_serializer *_serializer, int ndpi_serialize_uint32_uint64(ndpi_serializer *_serializer, u_int32_t key, u_int64_t value) { ndpi_private_serializer *serializer = (ndpi_private_serializer*)_serializer; - u_int32_t buff_diff = serializer->buffer_size - serializer->status.size_used; + u_int32_t buff_diff = serializer->buffer.size - serializer->status.buffer.size_used; u_int16_t needed = sizeof(u_int8_t) /* type */ + sizeof(u_int32_t) /* key */ + @@ -632,20 +764,31 @@ int ndpi_serialize_uint32_uint64(ndpi_serializer *_serializer, needed += 32; if(buff_diff < needed) { - if(ndpi_extend_serializer_buffer(_serializer, needed - buff_diff) < 0) + if(ndpi_extend_serializer_buffer(&serializer->buffer, needed - buff_diff) < 0) return(-1); - buff_diff = serializer->buffer_size - serializer->status.size_used; + buff_diff = serializer->buffer.size - serializer->status.buffer.size_used; } if(serializer->fmt == ndpi_serialization_format_json) { ndpi_serialize_json_pre(_serializer); - serializer->status.size_used += snprintf((char *) &serializer->buffer[serializer->status.size_used], buff_diff, - "\"%u\":%llu", key, (unsigned long long)value); + + if (!(serializer->status.flags & NDPI_SERIALIZER_STATUS_LIST)) { + serializer->status.buffer.size_used += snprintf((char *) + &serializer->buffer.data[serializer->status.buffer.size_used], buff_diff, + "\"%u\":", key); + buff_diff = serializer->buffer.size - serializer->status.buffer.size_used; + } + + serializer->status.buffer.size_used += snprintf((char *) + &serializer->buffer.data[serializer->status.buffer.size_used], buff_diff, + "%llu", (unsigned long long)value); + ndpi_serialize_json_post(_serializer); } else if(serializer->fmt == ndpi_serialization_format_csv) { - serializer->status.size_used += snprintf((char *) &serializer->buffer[serializer->status.size_used], buff_diff, + if (ndpi_serializer_header_uint32(serializer, key) < 0) return(-1); + serializer->status.buffer.size_used += snprintf((char *) &serializer->buffer.data[serializer->status.buffer.size_used], buff_diff, "%s%llu", - (serializer->status.size_used > 0) ? serializer->csv_separator : "", + (serializer->status.buffer.size_used > 0) ? serializer->csv_separator : "", (unsigned long long)value); } else { if(value <= 0xffffffff) { @@ -653,7 +796,7 @@ int ndpi_serialize_uint32_uint64(ndpi_serializer *_serializer, } else { ndpi_serialization_type kt; u_int8_t type = 0; - u_int32_t type_offset = serializer->status.size_used++; + u_int32_t type_offset = serializer->status.buffer.size_used++; kt = ndpi_serialize_key_uint32(serializer, key); type = (kt << 4); @@ -661,7 +804,7 @@ int ndpi_serialize_uint32_uint64(ndpi_serializer *_serializer, ndpi_serialize_single_uint64(serializer, value); type |= ndpi_serialization_uint64; - serializer->buffer[type_offset] = type; + serializer->buffer.data[type_offset] = type; } } @@ -674,7 +817,7 @@ int ndpi_serialize_uint32_uint64(ndpi_serializer *_serializer, int ndpi_serialize_uint32_int32(ndpi_serializer *_serializer, u_int32_t key, int32_t value) { ndpi_private_serializer *serializer = (ndpi_private_serializer*)_serializer; - u_int32_t buff_diff = serializer->buffer_size - serializer->status.size_used; + u_int32_t buff_diff = serializer->buffer.size - serializer->status.buffer.size_used; u_int16_t needed = sizeof(u_int8_t) /* type */ + sizeof(u_int32_t) /* key */ + @@ -684,23 +827,34 @@ int ndpi_serialize_uint32_int32(ndpi_serializer *_serializer, needed += 24; if(buff_diff < needed) { - if(ndpi_extend_serializer_buffer(_serializer, needed - buff_diff) < 0) + if(ndpi_extend_serializer_buffer(&serializer->buffer, needed - buff_diff) < 0) return(-1); - buff_diff = serializer->buffer_size - serializer->status.size_used; + buff_diff = serializer->buffer.size - serializer->status.buffer.size_used; } if(serializer->fmt == ndpi_serialization_format_json) { ndpi_serialize_json_pre(_serializer); - serializer->status.size_used += snprintf((char *) &serializer->buffer[serializer->status.size_used], buff_diff, - "\"%u\":%d", key, value); + + if (!(serializer->status.flags & NDPI_SERIALIZER_STATUS_LIST)) { + serializer->status.buffer.size_used += snprintf((char *) + &serializer->buffer.data[serializer->status.buffer.size_used], + buff_diff, "\"%u\":", key); + buff_diff = serializer->buffer.size - serializer->status.buffer.size_used; + } + + serializer->status.buffer.size_used += snprintf((char *) + &serializer->buffer.data[serializer->status.buffer.size_used], + buff_diff, "%d", value); + ndpi_serialize_json_post(_serializer); } else if(serializer->fmt == ndpi_serialization_format_csv) { - serializer->status.size_used += snprintf((char *) &serializer->buffer[serializer->status.size_used], buff_diff, - "%s%d", (serializer->status.size_used > 0) ? serializer->csv_separator : "", value); + if (ndpi_serializer_header_uint32(serializer, key) < 0) return(-1); + serializer->status.buffer.size_used += snprintf((char *) &serializer->buffer.data[serializer->status.buffer.size_used], buff_diff, + "%s%d", (serializer->status.buffer.size_used > 0) ? serializer->csv_separator : "", value); } else { ndpi_serialization_type kt; u_int8_t type = 0; - u_int32_t type_offset = serializer->status.size_used++; + u_int32_t type_offset = serializer->status.buffer.size_used++; kt = ndpi_serialize_key_uint32(serializer, key); type = (kt << 4); @@ -716,7 +870,7 @@ int ndpi_serialize_uint32_int32(ndpi_serializer *_serializer, type |= ndpi_serialization_int32; } - serializer->buffer[type_offset] = type; + serializer->buffer.data[type_offset] = type; } serializer->status.flags |= NDPI_SERIALIZER_STATUS_NOT_EMPTY; @@ -728,7 +882,7 @@ int ndpi_serialize_uint32_int32(ndpi_serializer *_serializer, int ndpi_serialize_uint32_int64(ndpi_serializer *_serializer, u_int32_t key, int64_t value) { ndpi_private_serializer *serializer = (ndpi_private_serializer*)_serializer; - u_int32_t buff_diff = serializer->buffer_size - serializer->status.size_used; + u_int32_t buff_diff = serializer->buffer.size - serializer->status.buffer.size_used; u_int16_t needed = sizeof(u_int8_t) /* type */ + sizeof(u_int32_t) /* key */ + @@ -738,20 +892,31 @@ int ndpi_serialize_uint32_int64(ndpi_serializer *_serializer, needed += 32; if(buff_diff < needed) { - if(ndpi_extend_serializer_buffer(_serializer, needed - buff_diff) < 0) + if(ndpi_extend_serializer_buffer(&serializer->buffer, needed - buff_diff) < 0) return(-1); - buff_diff = serializer->buffer_size - serializer->status.size_used; + buff_diff = serializer->buffer.size - serializer->status.buffer.size_used; } if(serializer->fmt == ndpi_serialization_format_json) { ndpi_serialize_json_pre(_serializer); - serializer->status.size_used += snprintf((char *) &serializer->buffer[serializer->status.size_used], buff_diff, - "\"%u\":%lld", key, (long long int)value); + + if (!(serializer->status.flags & NDPI_SERIALIZER_STATUS_LIST)) { + serializer->status.buffer.size_used += snprintf((char *) + &serializer->buffer.data[serializer->status.buffer.size_used], + buff_diff, "\"%u\":", key); + buff_diff = serializer->buffer.size - serializer->status.buffer.size_used; + } + + serializer->status.buffer.size_used += snprintf((char *) + &serializer->buffer.data[serializer->status.buffer.size_used], + buff_diff, "%lld", (long long int)value); + ndpi_serialize_json_post(_serializer); } else if(serializer->fmt == ndpi_serialization_format_csv) { - serializer->status.size_used += snprintf((char *) &serializer->buffer[serializer->status.size_used], buff_diff, + if (ndpi_serializer_header_uint32(serializer, key) < 0) return(-1); + serializer->status.buffer.size_used += snprintf((char *) &serializer->buffer.data[serializer->status.buffer.size_used], buff_diff, "%s%lld", - (serializer->status.size_used > 0) ? serializer->csv_separator : "", + (serializer->status.buffer.size_used > 0) ? serializer->csv_separator : "", (long long int)value); } @@ -761,7 +926,7 @@ int ndpi_serialize_uint32_int64(ndpi_serializer *_serializer, } else { ndpi_serialization_type kt; u_int8_t type = 0; - u_int32_t type_offset = serializer->status.size_used++; + u_int32_t type_offset = serializer->status.buffer.size_used++; kt = ndpi_serialize_key_uint32(serializer, key); type = (kt << 4); @@ -769,7 +934,7 @@ int ndpi_serialize_uint32_int64(ndpi_serializer *_serializer, ndpi_serialize_single_uint64(serializer, value); type |= ndpi_serialization_int64; - serializer->buffer[type_offset] = type; + serializer->buffer.data[type_offset] = type; } } @@ -783,7 +948,7 @@ int ndpi_serialize_uint32_float(ndpi_serializer *_serializer, u_int32_t key, float value, const char *format /* e.f. "%.2f" */) { ndpi_private_serializer *serializer = (ndpi_private_serializer*)_serializer; - u_int32_t buff_diff = serializer->buffer_size - serializer->status.size_used; + u_int32_t buff_diff = serializer->buffer.size - serializer->status.buffer.size_used; u_int16_t needed = sizeof(u_int8_t) /* type */ + sizeof(u_int32_t) /* key */ + @@ -793,25 +958,32 @@ int ndpi_serialize_uint32_float(ndpi_serializer *_serializer, needed += 32; if(buff_diff < needed) { - if(ndpi_extend_serializer_buffer(_serializer, needed - buff_diff) < 0) + if(ndpi_extend_serializer_buffer(&serializer->buffer, needed - buff_diff) < 0) return(-1); - buff_diff = serializer->buffer_size - serializer->status.size_used; + buff_diff = serializer->buffer.size - serializer->status.buffer.size_used; } if(serializer->fmt == ndpi_serialization_format_json) { ndpi_serialize_json_pre(_serializer); - serializer->status.size_used += snprintf((char *) &serializer->buffer[serializer->status.size_used], buff_diff, "\"%u\":", key); - serializer->status.size_used += snprintf((char *) &serializer->buffer[serializer->status.size_used], buff_diff, format, value); + + if (!(serializer->status.flags & NDPI_SERIALIZER_STATUS_LIST)) { + serializer->status.buffer.size_used += snprintf((char *) &serializer->buffer.data[serializer->status.buffer.size_used], buff_diff, "\"%u\":", key); + buff_diff = serializer->buffer.size - serializer->status.buffer.size_used; + } + + serializer->status.buffer.size_used += snprintf((char *) &serializer->buffer.data[serializer->status.buffer.size_used], buff_diff, format, value); + ndpi_serialize_json_post(_serializer); } else if(serializer->fmt == ndpi_serialization_format_csv) { - serializer->status.size_used += snprintf((char *) &serializer->buffer[serializer->status.size_used], buff_diff, "%s", - (serializer->status.size_used > 0) ? serializer->csv_separator : ""); - serializer->status.size_used += snprintf((char *) &serializer->buffer[serializer->status.size_used], buff_diff, format, value); + if (ndpi_serializer_header_uint32(serializer, key) < 0) return(-1); + serializer->status.buffer.size_used += snprintf((char *) &serializer->buffer.data[serializer->status.buffer.size_used], buff_diff, "%s", + (serializer->status.buffer.size_used > 0) ? serializer->csv_separator : ""); + serializer->status.buffer.size_used += snprintf((char *) &serializer->buffer.data[serializer->status.buffer.size_used], buff_diff, format, value); } else { ndpi_serialization_type kt; u_int8_t type = 0; - u_int32_t type_offset = serializer->status.size_used++; + u_int32_t type_offset = serializer->status.buffer.size_used++; kt = ndpi_serialize_key_uint32(serializer, key); type = (kt << 4); @@ -819,7 +991,7 @@ int ndpi_serialize_uint32_float(ndpi_serializer *_serializer, ndpi_serialize_single_float(serializer, value); type |= ndpi_serialization_float; - serializer->buffer[type_offset] = type; + serializer->buffer.data[type_offset] = type; } serializer->status.flags |= NDPI_SERIALIZER_STATUS_NOT_EMPTY; @@ -831,7 +1003,7 @@ int ndpi_serialize_uint32_float(ndpi_serializer *_serializer, static int ndpi_serialize_uint32_binary(ndpi_serializer *_serializer, u_int32_t key, const char *value, u_int16_t slen) { ndpi_private_serializer *serializer = (ndpi_private_serializer*)_serializer; - u_int32_t buff_diff = serializer->buffer_size - serializer->status.size_used; + u_int32_t buff_diff = serializer->buffer.size - serializer->status.buffer.size_used; u_int32_t needed = sizeof(u_int8_t) /* type */ + sizeof(u_int32_t) /* key */ + @@ -842,27 +1014,31 @@ static int ndpi_serialize_uint32_binary(ndpi_serializer *_serializer, needed += 24 + slen; if(buff_diff < needed) { - if(ndpi_extend_serializer_buffer(_serializer, needed - buff_diff) < 0) + if(ndpi_extend_serializer_buffer(&serializer->buffer, needed - buff_diff) < 0) return(-1); - buff_diff = serializer->buffer_size - serializer->status.size_used; + buff_diff = serializer->buffer.size - serializer->status.buffer.size_used; } if(serializer->fmt == ndpi_serialization_format_json) { ndpi_serialize_json_pre(_serializer); - serializer->status.size_used += snprintf((char *) &serializer->buffer[serializer->status.size_used], buff_diff, - "\"%u\":", key); - buff_diff = serializer->buffer_size - serializer->status.size_used; - serializer->status.size_used += ndpi_json_string_escape(value, slen, - (char *) &serializer->buffer[serializer->status.size_used], buff_diff); - buff_diff = serializer->buffer_size - serializer->status.size_used; + + if (!(serializer->status.flags & NDPI_SERIALIZER_STATUS_LIST)) { + serializer->status.buffer.size_used += snprintf((char *) &serializer->buffer.data[serializer->status.buffer.size_used], + buff_diff, "\"%u\":", key); + buff_diff = serializer->buffer.size - serializer->status.buffer.size_used; + } + serializer->status.buffer.size_used += ndpi_json_string_escape(value, slen, + (char *) &serializer->buffer.data[serializer->status.buffer.size_used], buff_diff); + buff_diff = serializer->buffer.size - serializer->status.buffer.size_used; ndpi_serialize_json_post(_serializer); } else if(serializer->fmt == ndpi_serialization_format_csv) { - serializer->status.size_used += snprintf((char *) &serializer->buffer[serializer->status.size_used], buff_diff, - "%s%s", (serializer->status.size_used > 0) ? serializer->csv_separator : "", value); + if (ndpi_serializer_header_uint32(serializer, key) < 0) return(-1); + serializer->status.buffer.size_used += snprintf((char *) &serializer->buffer.data[serializer->status.buffer.size_used], buff_diff, + "%s%s", (serializer->status.buffer.size_used > 0) ? serializer->csv_separator : "", value); } else { ndpi_serialization_type kt; u_int8_t type = 0; - u_int32_t type_offset = serializer->status.size_used++; + u_int32_t type_offset = serializer->status.buffer.size_used++; kt = ndpi_serialize_key_uint32(serializer, key); type = (kt << 4); @@ -870,7 +1046,7 @@ static int ndpi_serialize_uint32_binary(ndpi_serializer *_serializer, ndpi_serialize_single_string(serializer, value, slen); type |= ndpi_serialization_string; - serializer->buffer[type_offset] = type; + serializer->buffer.data[type_offset] = type; } serializer->status.flags |= NDPI_SERIALIZER_STATUS_NOT_EMPTY; @@ -890,7 +1066,7 @@ int ndpi_serialize_uint32_string(ndpi_serializer *_serializer, int ndpi_serialize_uint32_boolean(ndpi_serializer *_serializer, u_int32_t key, u_int8_t value) { ndpi_private_serializer *serializer = (ndpi_private_serializer*)_serializer; - u_int32_t buff_diff = serializer->buffer_size - serializer->status.size_used; + u_int32_t buff_diff = serializer->buffer.size - serializer->status.buffer.size_used; u_int32_t needed = 24; if(serializer->fmt != ndpi_serialization_format_json && @@ -898,19 +1074,30 @@ int ndpi_serialize_uint32_boolean(ndpi_serializer *_serializer, return -1; if(buff_diff < needed) { - if(ndpi_extend_serializer_buffer(_serializer, needed - buff_diff) < 0) + if(ndpi_extend_serializer_buffer(&serializer->buffer, needed - buff_diff) < 0) return(-1); - buff_diff = serializer->buffer_size - serializer->status.size_used; + buff_diff = serializer->buffer.size - serializer->status.buffer.size_used; } if(serializer->fmt == ndpi_serialization_format_json) { ndpi_serialize_json_pre(_serializer); - serializer->status.size_used += snprintf((char *) &serializer->buffer[serializer->status.size_used], buff_diff, - "\"%u\":%s", key, value ? "true" : "false"); + + if (!(serializer->status.flags & NDPI_SERIALIZER_STATUS_LIST)) { + serializer->status.buffer.size_used += snprintf((char *) + &serializer->buffer.data[serializer->status.buffer.size_used], + buff_diff, "\"%u\":", key); + buff_diff = serializer->buffer.size - serializer->status.buffer.size_used; + } + + serializer->status.buffer.size_used += snprintf((char *) + &serializer->buffer.data[serializer->status.buffer.size_used], + buff_diff, "%s", value ? "true" : "false"); + ndpi_serialize_json_post(_serializer); } else if(serializer->fmt == ndpi_serialization_format_csv) { - serializer->status.size_used += snprintf((char *) &serializer->buffer[serializer->status.size_used], buff_diff, - "%s%s", (serializer->status.size_used > 0) ? serializer->csv_separator : "", + if (ndpi_serializer_header_uint32(serializer, key) < 0) return(-1); + serializer->status.buffer.size_used += snprintf((char *) &serializer->buffer.data[serializer->status.buffer.size_used], buff_diff, + "%s%s", (serializer->status.buffer.size_used > 0) ? serializer->csv_separator : "", value ? "true" : "false"); } @@ -924,7 +1111,7 @@ static int ndpi_serialize_binary_int32(ndpi_serializer *_serializer, const char *key, u_int16_t klen, int32_t value) { ndpi_private_serializer *serializer = (ndpi_private_serializer*)_serializer; - u_int32_t buff_diff = serializer->buffer_size - serializer->status.size_used; + u_int32_t buff_diff = serializer->buffer.size - serializer->status.buffer.size_used; u_int32_t needed; if(ndpi_is_number(key, klen)) @@ -940,33 +1127,42 @@ static int ndpi_serialize_binary_int32(ndpi_serializer *_serializer, needed += 16 + klen; if(buff_diff < needed) { - if(ndpi_extend_serializer_buffer(_serializer, needed - buff_diff) < 0) + if(ndpi_extend_serializer_buffer(&serializer->buffer, needed - buff_diff) < 0) return(-1); - buff_diff = serializer->buffer_size - serializer->status.size_used; + buff_diff = serializer->buffer.size - serializer->status.buffer.size_used; } if(serializer->fmt == ndpi_serialization_format_json) { ndpi_serialize_json_pre(_serializer); - serializer->status.size_used += ndpi_json_string_escape(key, klen, - (char *) &serializer->buffer[serializer->status.size_used], buff_diff); - buff_diff = serializer->buffer_size - serializer->status.size_used; - serializer->status.size_used += snprintf((char *) &serializer->buffer[serializer->status.size_used], buff_diff, - ":%d", value); + + if (!(serializer->status.flags & NDPI_SERIALIZER_STATUS_LIST)) { + serializer->status.buffer.size_used += ndpi_json_string_escape(key, klen, + (char *) &serializer->buffer.data[serializer->status.buffer.size_used], buff_diff); + buff_diff = serializer->buffer.size - serializer->status.buffer.size_used; + serializer->status.buffer.size_used += snprintf((char *) + &serializer->buffer.data[serializer->status.buffer.size_used], buff_diff, ":"); + buff_diff = serializer->buffer.size - serializer->status.buffer.size_used; + } + + serializer->status.buffer.size_used += snprintf((char *) + &serializer->buffer.data[serializer->status.buffer.size_used], buff_diff, "%d", value); + ndpi_serialize_json_post(_serializer); } else if(serializer->fmt == ndpi_serialization_format_csv) { - serializer->status.size_used += snprintf((char *) &serializer->buffer[serializer->status.size_used], buff_diff, - "%s%d", (serializer->status.size_used > 0) ? serializer->csv_separator : "", value); + if (ndpi_serializer_header_string(serializer, key, klen) < 0) return(-1); + serializer->status.buffer.size_used += snprintf((char *) &serializer->buffer.data[serializer->status.buffer.size_used], buff_diff, + "%s%d", (serializer->status.buffer.size_used > 0) ? serializer->csv_separator : "", value); } else { if(value <= 127 && value >= -128) { - serializer->buffer[serializer->status.size_used++] = (ndpi_serialization_string << 4) | ndpi_serialization_int8; + serializer->buffer.data[serializer->status.buffer.size_used++] = (ndpi_serialization_string << 4) | ndpi_serialization_int8; ndpi_serialize_single_string(serializer, key, klen); ndpi_serialize_single_uint8(serializer, value); } else if(value <= 32767 && value >= -32768) { - serializer->buffer[serializer->status.size_used++] = (ndpi_serialization_string << 4) | ndpi_serialization_int16; + serializer->buffer.data[serializer->status.buffer.size_used++] = (ndpi_serialization_string << 4) | ndpi_serialization_int16; ndpi_serialize_single_string(serializer, key, klen); ndpi_serialize_single_uint16(serializer, value); } else { - serializer->buffer[serializer->status.size_used++] = (ndpi_serialization_string << 4) | ndpi_serialization_int32; + serializer->buffer.data[serializer->status.buffer.size_used++] = (ndpi_serialization_string << 4) | ndpi_serialization_int32; ndpi_serialize_single_string(serializer, key, klen); ndpi_serialize_single_uint32(serializer, value); } @@ -990,7 +1186,7 @@ int ndpi_serialize_binary_int64(ndpi_serializer *_serializer, const char *key, u_int16_t klen, int64_t value) { ndpi_private_serializer *serializer = (ndpi_private_serializer*)_serializer; - u_int32_t buff_diff = serializer->buffer_size - serializer->status.size_used; + u_int32_t buff_diff = serializer->buffer.size - serializer->status.buffer.size_used; u_int32_t needed; if(ndpi_is_number(key, klen)) @@ -1006,28 +1202,39 @@ int ndpi_serialize_binary_int64(ndpi_serializer *_serializer, needed += 16 + klen; if(buff_diff < needed) { - if(ndpi_extend_serializer_buffer(_serializer, needed - buff_diff) < 0) + if(ndpi_extend_serializer_buffer(&serializer->buffer, needed - buff_diff) < 0) return(-1); - buff_diff = serializer->buffer_size - serializer->status.size_used; + buff_diff = serializer->buffer.size - serializer->status.buffer.size_used; } if(serializer->fmt == ndpi_serialization_format_json) { ndpi_serialize_json_pre(_serializer); - serializer->status.size_used += ndpi_json_string_escape(key, klen, - (char *) &serializer->buffer[serializer->status.size_used], buff_diff); - buff_diff = serializer->buffer_size - serializer->status.size_used; - serializer->status.size_used += snprintf((char *) &serializer->buffer[serializer->status.size_used], buff_diff, - ":%lld", (long long int)value); + + if (!(serializer->status.flags & NDPI_SERIALIZER_STATUS_LIST)) { + serializer->status.buffer.size_used += ndpi_json_string_escape(key, klen, + (char *) &serializer->buffer.data[serializer->status.buffer.size_used], buff_diff); + buff_diff = serializer->buffer.size - serializer->status.buffer.size_used; + serializer->status.buffer.size_used += snprintf((char *) + &serializer->buffer.data[serializer->status.buffer.size_used], buff_diff, + ":"); + buff_diff = serializer->buffer.size - serializer->status.buffer.size_used; + } + + serializer->status.buffer.size_used += snprintf((char *) + &serializer->buffer.data[serializer->status.buffer.size_used], buff_diff, + "%lld", (long long int)value); + ndpi_serialize_json_post(_serializer); } else if(serializer->fmt == ndpi_serialization_format_csv) { - serializer->status.size_used += snprintf((char *) &serializer->buffer[serializer->status.size_used], buff_diff, - "%s%lld", (serializer->status.size_used > 0) ? serializer->csv_separator : "", + if (ndpi_serializer_header_string(serializer, key, klen) < 0) return(-1); + serializer->status.buffer.size_used += snprintf((char *) &serializer->buffer.data[serializer->status.buffer.size_used], buff_diff, + "%s%lld", (serializer->status.buffer.size_used > 0) ? serializer->csv_separator : "", (long long int)value); } else { if ((value & 0xFFFFFFFF) == value) { return(ndpi_serialize_string_int32(_serializer, key, value)); } else { - serializer->buffer[serializer->status.size_used++] = (ndpi_serialization_string << 4) | ndpi_serialization_int64; + serializer->buffer.data[serializer->status.buffer.size_used++] = (ndpi_serialization_string << 4) | ndpi_serialization_int64; ndpi_serialize_single_string(serializer, key, klen); ndpi_serialize_single_uint32(serializer, value); } @@ -1049,7 +1256,7 @@ int ndpi_serialize_string_int64(ndpi_serializer *_serializer, static int ndpi_serialize_binary_uint32(ndpi_serializer *_serializer, const char *key, u_int16_t klen, u_int32_t value) { ndpi_private_serializer *serializer = (ndpi_private_serializer*)_serializer; - u_int32_t buff_diff = serializer->buffer_size - serializer->status.size_used; + u_int32_t buff_diff = serializer->buffer.size - serializer->status.buffer.size_used; u_int32_t needed; if(ndpi_is_number(key, klen)) @@ -1065,33 +1272,45 @@ static int ndpi_serialize_binary_uint32(ndpi_serializer *_serializer, needed += 16 + klen; if(buff_diff < needed) { - if(ndpi_extend_serializer_buffer(_serializer, needed - buff_diff) < 0) + if(ndpi_extend_serializer_buffer(&serializer->buffer, needed - buff_diff) < 0) return(-1); - buff_diff = serializer->buffer_size - serializer->status.size_used; + buff_diff = serializer->buffer.size - serializer->status.buffer.size_used; } if(serializer->fmt == ndpi_serialization_format_json) { ndpi_serialize_json_pre(_serializer); - serializer->status.size_used += ndpi_json_string_escape(key, klen, - (char *) &serializer->buffer[serializer->status.size_used], buff_diff); - buff_diff = serializer->buffer_size - serializer->status.size_used; - serializer->status.size_used += snprintf((char *) &serializer->buffer[serializer->status.size_used], buff_diff, - ":%u", value); + + if (!(serializer->status.flags & NDPI_SERIALIZER_STATUS_LIST)) { + serializer->status.buffer.size_used += ndpi_json_string_escape(key, klen, + (char *) &serializer->buffer.data[serializer->status.buffer.size_used], buff_diff); + buff_diff = serializer->buffer.size - serializer->status.buffer.size_used; + + serializer->status.buffer.size_used += snprintf((char *) + &serializer->buffer.data[serializer->status.buffer.size_used], + buff_diff, ":"); + buff_diff = serializer->buffer.size - serializer->status.buffer.size_used; + } + + serializer->status.buffer.size_used += snprintf((char *) + &serializer->buffer.data[serializer->status.buffer.size_used], + buff_diff, "%u", value); + ndpi_serialize_json_post(_serializer); } else if(serializer->fmt == ndpi_serialization_format_csv) { - serializer->status.size_used += snprintf((char *) &serializer->buffer[serializer->status.size_used], buff_diff, - "%s%u", (serializer->status.size_used > 0) ? serializer->csv_separator : "", value); + if (ndpi_serializer_header_string(serializer, key, klen) < 0) return(-1); + serializer->status.buffer.size_used += snprintf((char *) &serializer->buffer.data[serializer->status.buffer.size_used], buff_diff, + "%s%u", (serializer->status.buffer.size_used > 0) ? serializer->csv_separator : "", value); } else { if(value <= 0xff) { - serializer->buffer[serializer->status.size_used++] = (ndpi_serialization_string << 4) | ndpi_serialization_uint8; + serializer->buffer.data[serializer->status.buffer.size_used++] = (ndpi_serialization_string << 4) | ndpi_serialization_uint8; ndpi_serialize_single_string(serializer, key, klen); ndpi_serialize_single_uint8(serializer, value); } else if(value <= 0xffff) { - serializer->buffer[serializer->status.size_used++] = (ndpi_serialization_string << 4) | ndpi_serialization_uint16; + serializer->buffer.data[serializer->status.buffer.size_used++] = (ndpi_serialization_string << 4) | ndpi_serialization_uint16; ndpi_serialize_single_string(serializer, key, klen); ndpi_serialize_single_uint16(serializer, value); } else { - serializer->buffer[serializer->status.size_used++] = (ndpi_serialization_string << 4) | ndpi_serialization_uint32; + serializer->buffer.data[serializer->status.buffer.size_used++] = (ndpi_serialization_string << 4) | ndpi_serialization_uint32; ndpi_serialize_single_string(serializer, key, klen); ndpi_serialize_single_uint32(serializer, value); } @@ -1136,7 +1355,7 @@ static int ndpi_serialize_binary_uint64(ndpi_serializer *_serializer, const char *key, u_int16_t klen, u_int64_t value) { ndpi_private_serializer *serializer = (ndpi_private_serializer*)_serializer; - u_int32_t buff_diff = serializer->buffer_size - serializer->status.size_used; + u_int32_t buff_diff = serializer->buffer.size - serializer->status.buffer.size_used; u_int32_t needed; if(ndpi_is_number(key, klen)) @@ -1152,28 +1371,39 @@ static int ndpi_serialize_binary_uint64(ndpi_serializer *_serializer, needed += 32 + klen; if(buff_diff < needed) { - if(ndpi_extend_serializer_buffer(_serializer, needed - buff_diff) < 0) + if(ndpi_extend_serializer_buffer(&serializer->buffer, needed - buff_diff) < 0) return(-1); - buff_diff = serializer->buffer_size - serializer->status.size_used; + buff_diff = serializer->buffer.size - serializer->status.buffer.size_used; } if(serializer->fmt == ndpi_serialization_format_json) { ndpi_serialize_json_pre(_serializer); - serializer->status.size_used += ndpi_json_string_escape(key, klen, - (char *) &serializer->buffer[serializer->status.size_used], buff_diff); - buff_diff = serializer->buffer_size - serializer->status.size_used; - serializer->status.size_used += snprintf((char *) &serializer->buffer[serializer->status.size_used], buff_diff, - ":%llu", (unsigned long long)value); + + if (!(serializer->status.flags & NDPI_SERIALIZER_STATUS_LIST)) { + serializer->status.buffer.size_used += ndpi_json_string_escape(key, klen, + (char *) &serializer->buffer.data[serializer->status.buffer.size_used], buff_diff); + buff_diff = serializer->buffer.size - serializer->status.buffer.size_used; + serializer->status.buffer.size_used += snprintf((char *) + &serializer->buffer.data[serializer->status.buffer.size_used], buff_diff, + ":"); + buff_diff = serializer->buffer.size - serializer->status.buffer.size_used; + } + + serializer->status.buffer.size_used += snprintf((char *) + &serializer->buffer.data[serializer->status.buffer.size_used], buff_diff, + "%llu", (unsigned long long)value); + ndpi_serialize_json_post(_serializer); } else if(serializer->fmt == ndpi_serialization_format_csv) { - serializer->status.size_used += snprintf((char *) &serializer->buffer[serializer->status.size_used], buff_diff, - "%s%llu", (serializer->status.size_used > 0) ? serializer->csv_separator : "", + if (ndpi_serializer_header_string(serializer, key, klen) < 0) return(-1); + serializer->status.buffer.size_used += snprintf((char *) &serializer->buffer.data[serializer->status.buffer.size_used], buff_diff, + "%s%llu", (serializer->status.buffer.size_used > 0) ? serializer->csv_separator : "", (unsigned long long)value); } else { if(value <= 0xffffffff) { return(ndpi_serialize_string_uint32(_serializer, key, value)); } else { - serializer->buffer[serializer->status.size_used++] = (ndpi_serialization_string << 4) | ndpi_serialization_uint64; + serializer->buffer.data[serializer->status.buffer.size_used++] = (ndpi_serialization_string << 4) | ndpi_serialization_uint64; ndpi_serialize_single_string(serializer, key, klen); ndpi_serialize_single_uint64(serializer, value); } @@ -1198,7 +1428,7 @@ static int ndpi_serialize_binary_float(ndpi_serializer *_serializer, float value, const char *format /* e.f. "%.2f" */) { ndpi_private_serializer *serializer = (ndpi_private_serializer*)_serializer; - u_int32_t buff_diff = serializer->buffer_size - serializer->status.size_used; + u_int32_t buff_diff = serializer->buffer.size - serializer->status.buffer.size_used; u_int32_t needed; if(ndpi_is_number(key, klen)) @@ -1214,30 +1444,32 @@ static int ndpi_serialize_binary_float(ndpi_serializer *_serializer, needed += 32 + klen; if(buff_diff < needed) { - if(ndpi_extend_serializer_buffer(_serializer, needed - buff_diff) < 0) + if(ndpi_extend_serializer_buffer(&serializer->buffer, needed - buff_diff) < 0) return(-1); - buff_diff = serializer->buffer_size - serializer->status.size_used; + buff_diff = serializer->buffer.size - serializer->status.buffer.size_used; } if(serializer->fmt == ndpi_serialization_format_json) { ndpi_serialize_json_pre(_serializer); - serializer->status.size_used += ndpi_json_string_escape(key, klen, - (char *) &serializer->buffer[serializer->status.size_used], buff_diff); - buff_diff = serializer->buffer_size - serializer->status.size_used; - serializer->buffer[serializer->status.size_used] = ':'; - serializer->status.size_used++; + if (!(serializer->status.flags & NDPI_SERIALIZER_STATUS_LIST)) { + serializer->status.buffer.size_used += ndpi_json_string_escape(key, klen, + (char *) &serializer->buffer.data[serializer->status.buffer.size_used], buff_diff); + buff_diff = serializer->buffer.size - serializer->status.buffer.size_used; + serializer->buffer.data[serializer->status.buffer.size_used] = ':'; + serializer->status.buffer.size_used++; + } - serializer->status.size_used += snprintf((char *) &serializer->buffer[serializer->status.size_used], buff_diff, format, value); + serializer->status.buffer.size_used += snprintf((char *) &serializer->buffer.data[serializer->status.buffer.size_used], buff_diff, format, value); ndpi_serialize_json_post(_serializer); } else if(serializer->fmt == ndpi_serialization_format_csv) { - if(serializer->status.size_used > 0) - serializer->status.size_used += snprintf((char *) &serializer->buffer[serializer->status.size_used], buff_diff, "%s", serializer->csv_separator); - - serializer->status.size_used += snprintf((char *) &serializer->buffer[serializer->status.size_used], buff_diff, format, value); + if (ndpi_serializer_header_string(serializer, key, klen) < 0) return(-1); + if(serializer->status.buffer.size_used > 0) + serializer->status.buffer.size_used += snprintf((char *) &serializer->buffer.data[serializer->status.buffer.size_used], buff_diff, "%s", serializer->csv_separator); + serializer->status.buffer.size_used += snprintf((char *) &serializer->buffer.data[serializer->status.buffer.size_used], buff_diff, format, value); } else { - serializer->buffer[serializer->status.size_used++] = (ndpi_serialization_string << 4) | ndpi_serialization_float; + serializer->buffer.data[serializer->status.buffer.size_used++] = (ndpi_serialization_string << 4) | ndpi_serialization_float; ndpi_serialize_single_string(serializer, key, klen); ndpi_serialize_single_float(serializer, value); @@ -1266,7 +1498,7 @@ static int ndpi_serialize_binary_raw(ndpi_serializer *_serializer, u_int16_t vlen, u_int8_t escape) { ndpi_private_serializer *serializer = (ndpi_private_serializer*)_serializer; - u_int32_t buff_diff = serializer->buffer_size - serializer->status.size_used; + u_int32_t buff_diff = serializer->buffer.size - serializer->status.buffer.size_used; u_int32_t needed; needed = @@ -1280,32 +1512,37 @@ static int ndpi_serialize_binary_raw(ndpi_serializer *_serializer, needed += 16 + klen + vlen; if(buff_diff < needed) { - if(ndpi_extend_serializer_buffer(_serializer, needed - buff_diff) < 0) + if(ndpi_extend_serializer_buffer(&serializer->buffer, needed - buff_diff) < 0) return(-1); - buff_diff = serializer->buffer_size - serializer->status.size_used; + buff_diff = serializer->buffer.size - serializer->status.buffer.size_used; } if(serializer->fmt == ndpi_serialization_format_json) { ndpi_serialize_json_pre(_serializer); - serializer->status.size_used += ndpi_json_string_escape(key, klen, - (char *) &serializer->buffer[serializer->status.size_used], buff_diff); - buff_diff = serializer->buffer_size - serializer->status.size_used; - serializer->status.size_used += snprintf((char *) &serializer->buffer[serializer->status.size_used], buff_diff, ":"); - buff_diff = serializer->buffer_size - serializer->status.size_used; + + if (!(serializer->status.flags & NDPI_SERIALIZER_STATUS_LIST)) { + serializer->status.buffer.size_used += ndpi_json_string_escape(key, klen, + (char *) &serializer->buffer.data[serializer->status.buffer.size_used], buff_diff); + buff_diff = serializer->buffer.size - serializer->status.buffer.size_used; + serializer->status.buffer.size_used += snprintf((char *) &serializer->buffer.data[serializer->status.buffer.size_used], buff_diff, ":"); + buff_diff = serializer->buffer.size - serializer->status.buffer.size_used; + } if (escape) - serializer->status.size_used += ndpi_json_string_escape(value, vlen, - (char *) &serializer->buffer[serializer->status.size_used], buff_diff); + serializer->status.buffer.size_used += ndpi_json_string_escape(value, vlen, + (char *) &serializer->buffer.data[serializer->status.buffer.size_used], buff_diff); else - serializer->status.size_used += snprintf((char *) &serializer->buffer[serializer->status.size_used], buff_diff, + serializer->status.buffer.size_used += snprintf((char *) &serializer->buffer.data[serializer->status.buffer.size_used], buff_diff, value, vlen); + ndpi_serialize_json_post(_serializer); } else if(serializer->fmt == ndpi_serialization_format_csv) { - serializer->status.size_used += snprintf((char *) &serializer->buffer[serializer->status.size_used], buff_diff, + if (ndpi_serializer_header_string(serializer, key, klen) < 0) return(-1); + serializer->status.buffer.size_used += snprintf((char *) &serializer->buffer.data[serializer->status.buffer.size_used], buff_diff, "%s%s", ndpi_serialize_is_not_empty(_serializer) ? serializer->csv_separator : "", value); } else { - serializer->buffer[serializer->status.size_used++] = (ndpi_serialization_string << 4) | ndpi_serialization_string; + serializer->buffer.data[serializer->status.buffer.size_used++] = (ndpi_serialization_string << 4) | ndpi_serialization_string; ndpi_serialize_single_string(serializer, key, klen); ndpi_serialize_single_string(serializer, value, vlen); @@ -1362,7 +1599,7 @@ int ndpi_serialize_string_raw(ndpi_serializer *_serializer, int ndpi_serialize_string_boolean(ndpi_serializer *_serializer, const char *key, u_int8_t value) { ndpi_private_serializer *serializer = (ndpi_private_serializer*)_serializer; - u_int32_t buff_diff = serializer->buffer_size - serializer->status.size_used; + u_int32_t buff_diff = serializer->buffer.size - serializer->status.buffer.size_used; u_int16_t klen = strlen(key); u_int32_t needed; @@ -1376,21 +1613,31 @@ int ndpi_serialize_string_boolean(ndpi_serializer *_serializer, needed = klen + 16; if(buff_diff < needed) { - if(ndpi_extend_serializer_buffer(_serializer, needed - buff_diff) < 0) + if(ndpi_extend_serializer_buffer(&serializer->buffer, needed - buff_diff) < 0) return(-1); - buff_diff = serializer->buffer_size - serializer->status.size_used; + buff_diff = serializer->buffer.size - serializer->status.buffer.size_used; } if(serializer->fmt == ndpi_serialization_format_json) { ndpi_serialize_json_pre(_serializer); - serializer->status.size_used += ndpi_json_string_escape(key, klen, - (char *) &serializer->buffer[serializer->status.size_used], buff_diff); - buff_diff = serializer->buffer_size - serializer->status.size_used; - serializer->status.size_used += snprintf((char *) &serializer->buffer[serializer->status.size_used], buff_diff, ":%s", + + if (!(serializer->status.flags & NDPI_SERIALIZER_STATUS_LIST)) { + serializer->status.buffer.size_used += ndpi_json_string_escape(key, klen, + (char *) &serializer->buffer.data[serializer->status.buffer.size_used], buff_diff); + buff_diff = serializer->buffer.size - serializer->status.buffer.size_used; + serializer->status.buffer.size_used += snprintf((char *) + &serializer->buffer.data[serializer->status.buffer.size_used], buff_diff, ":"); + buff_diff = serializer->buffer.size - serializer->status.buffer.size_used; + } + + serializer->status.buffer.size_used += snprintf((char *) + &serializer->buffer.data[serializer->status.buffer.size_used], buff_diff, "%s", value ? "true" : "false"); + ndpi_serialize_json_post(_serializer); } else if(serializer->fmt == ndpi_serialization_format_csv) { - serializer->status.size_used += snprintf((char *) &serializer->buffer[serializer->status.size_used], buff_diff, + if (ndpi_serializer_header_string(serializer, key, strlen(key)) < 0) return(-1); + serializer->status.buffer.size_used += snprintf((char *) &serializer->buffer.data[serializer->status.buffer.size_used], buff_diff, "%s%s", ndpi_serialize_is_not_empty(_serializer) ? serializer->csv_separator : "", value ? "true" : "false"); } @@ -1401,11 +1648,63 @@ int ndpi_serialize_string_boolean(ndpi_serializer *_serializer, /* ********************************** */ +/* Serialize start of simple list (JSON only)*/ +int ndpi_serialize_start_of_list(ndpi_serializer *_serializer, + const char *key) { + ndpi_private_serializer *serializer = (ndpi_private_serializer*)_serializer; + u_int32_t buff_diff = serializer->buffer.size - serializer->status.buffer.size_used; + u_int32_t needed, klen = strlen(key); + + if(serializer->fmt != ndpi_serialization_format_json) + return(-1); + + needed = 16 + klen; + + if(buff_diff < needed) { + if(ndpi_extend_serializer_buffer(&serializer->buffer, needed - buff_diff) < 0) + return(-1); + buff_diff = serializer->buffer.size - serializer->status.buffer.size_used; + } + + ndpi_serialize_json_pre(_serializer); + + serializer->status.buffer.size_used += ndpi_json_string_escape(key, klen, + (char *) &serializer->buffer.data[serializer->status.buffer.size_used], buff_diff); + buff_diff = serializer->buffer.size - serializer->status.buffer.size_used; + + serializer->status.buffer.size_used += snprintf((char *) &serializer->buffer.data[serializer->status.buffer.size_used], buff_diff, ": ["); + + serializer->status.flags |= NDPI_SERIALIZER_STATUS_LIST | NDPI_SERIALIZER_STATUS_SOL; + + ndpi_serialize_json_post(_serializer); + + return(0); +} + +/* ********************************** */ + +/* Serialize start of simple list (JSON only)*/ +int ndpi_serialize_end_of_list(ndpi_serializer *_serializer) { + ndpi_private_serializer *serializer = (ndpi_private_serializer*)_serializer; + + if(serializer->fmt != ndpi_serialization_format_json) + return(-1); + + if(serializer->status.flags & NDPI_SERIALIZER_STATUS_SOL) /* Empty list */ + serializer->status.flags &= ~NDPI_SERIALIZER_STATUS_SOL; + + serializer->status.flags &= ~NDPI_SERIALIZER_STATUS_LIST; + + return(0); +} + +/* ********************************** */ + /* Serialize start of nested block (JSON only)*/ int ndpi_serialize_start_of_block(ndpi_serializer *_serializer, const char *key) { ndpi_private_serializer *serializer = (ndpi_private_serializer*)_serializer; - u_int32_t buff_diff = serializer->buffer_size - serializer->status.size_used; + u_int32_t buff_diff = serializer->buffer.size - serializer->status.buffer.size_used; u_int32_t needed, klen = strlen(key); if(serializer->fmt != ndpi_serialization_format_json) @@ -1414,17 +1713,17 @@ int ndpi_serialize_start_of_block(ndpi_serializer *_serializer, needed = 16 + klen; if(buff_diff < needed) { - if(ndpi_extend_serializer_buffer(_serializer, needed - buff_diff) < 0) + if(ndpi_extend_serializer_buffer(&serializer->buffer, needed - buff_diff) < 0) return(-1); - buff_diff = serializer->buffer_size - serializer->status.size_used; + buff_diff = serializer->buffer.size - serializer->status.buffer.size_used; } ndpi_serialize_json_pre(_serializer); - serializer->status.size_used += ndpi_json_string_escape(key, klen, - (char *) &serializer->buffer[serializer->status.size_used], buff_diff); - buff_diff = serializer->buffer_size - serializer->status.size_used; - serializer->status.size_used += snprintf((char *) &serializer->buffer[serializer->status.size_used], buff_diff, ": {"); - buff_diff = serializer->buffer_size - serializer->status.size_used; + serializer->status.buffer.size_used += ndpi_json_string_escape(key, klen, + (char *) &serializer->buffer.data[serializer->status.buffer.size_used], buff_diff); + buff_diff = serializer->buffer.size - serializer->status.buffer.size_used; + serializer->status.buffer.size_used += snprintf((char *) &serializer->buffer.data[serializer->status.buffer.size_used], buff_diff, ": {"); + buff_diff = serializer->buffer.size - serializer->status.buffer.size_used; ndpi_serialize_json_post(_serializer); serializer->status.flags |= NDPI_SERIALIZER_STATUS_SOB; @@ -1434,10 +1733,10 @@ int ndpi_serialize_start_of_block(ndpi_serializer *_serializer, /* ********************************** */ -/* Serialize start of nested block (JSON only)*/ +/* Serialize end of nested block (JSON only)*/ int ndpi_serialize_end_of_block(ndpi_serializer *_serializer) { ndpi_private_serializer *serializer = (ndpi_private_serializer*)_serializer; - u_int32_t buff_diff = serializer->buffer_size - serializer->status.size_used; + u_int32_t buff_diff = serializer->buffer.size - serializer->status.buffer.size_used; u_int32_t needed; if(serializer->fmt != ndpi_serialization_format_json) @@ -1446,15 +1745,15 @@ int ndpi_serialize_end_of_block(ndpi_serializer *_serializer) { needed = 4; if(buff_diff < needed) { - if(ndpi_extend_serializer_buffer(_serializer, needed - buff_diff) < 0) + if(ndpi_extend_serializer_buffer(&serializer->buffer, needed - buff_diff) < 0) return(-1); - buff_diff = serializer->buffer_size - serializer->status.size_used; + buff_diff = serializer->buffer.size - serializer->status.buffer.size_used; } if(serializer->status.flags & NDPI_SERIALIZER_STATUS_SOB) /* Empty block */ serializer->status.flags &= ~NDPI_SERIALIZER_STATUS_SOB; - // buff_diff = serializer->buffer_size - serializer->status.size_used; + // buff_diff = serializer->buffer.size - serializer->status.buffer.size_used; ndpi_serialize_json_post(_serializer); return(0); @@ -1488,10 +1787,10 @@ void ndpi_serializer_rollback_snapshot(ndpi_serializer *_serializer) { if(serializer->fmt == ndpi_serialization_format_json) { if(serializer->status.flags & NDPI_SERIALIZER_STATUS_ARRAY) { - serializer->buffer[serializer->status.size_used-1] = ']'; + serializer->buffer.data[serializer->status.buffer.size_used-1] = ']'; } else { - serializer->buffer[0] = ' '; - serializer->buffer[serializer->status.size_used-1] = '}'; + serializer->buffer.data[0] = ' '; + serializer->buffer.data[serializer->status.buffer.size_used-1] = '}'; } } } @@ -1508,13 +1807,13 @@ int ndpi_init_deserializer_buf(ndpi_deserializer *_deserializer, if(serialized_buffer_len < (2 * sizeof(u_int8_t))) return(-1); - deserializer->buffer = serialized_buffer; + deserializer->buffer.data = serialized_buffer; - if(deserializer->buffer[0] != 1) + if(deserializer->buffer.data[0] != 1) return(-2); /* Invalid version */ - deserializer->buffer_size = serialized_buffer_len; - deserializer->fmt = deserializer->buffer[1]; + deserializer->buffer.size = serialized_buffer_len; + deserializer->fmt = deserializer->buffer.data[1]; ndpi_reset_serializer(_deserializer); return(0); @@ -1527,8 +1826,8 @@ int ndpi_init_deserializer(ndpi_deserializer *deserializer, ndpi_private_serializer *serializer = (ndpi_private_serializer*)_serializer; return(ndpi_init_deserializer_buf(deserializer, - serializer->buffer, - serializer->status.size_used)); + serializer->buffer.data, + serializer->status.buffer.size_used)); } /* ********************************** */ @@ -1543,10 +1842,10 @@ ndpi_serialization_format ndpi_deserialize_get_format(ndpi_deserializer *_deseri static inline ndpi_serialization_type ndpi_deserialize_get_key_subtype(ndpi_private_deserializer *deserializer) { u_int8_t type; - if(deserializer->status.size_used >= deserializer->buffer_size) + if(deserializer->status.buffer.size_used >= deserializer->buffer.size) return(ndpi_serialization_unknown); - type = deserializer->buffer[deserializer->status.size_used]; + type = deserializer->buffer.data[deserializer->status.buffer.size_used]; return((ndpi_serialization_type) (type >> 4)); } @@ -1556,10 +1855,10 @@ static inline ndpi_serialization_type ndpi_deserialize_get_key_subtype(ndpi_priv static inline ndpi_serialization_type ndpi_deserialize_get_value_subtype(ndpi_private_deserializer *deserializer) { u_int8_t type; - if(deserializer->status.size_used >= deserializer->buffer_size) + if(deserializer->status.buffer.size_used >= deserializer->buffer.size) return(ndpi_serialization_unknown); - type = deserializer->buffer[deserializer->status.size_used]; + type = deserializer->buffer.data[deserializer->status.buffer.size_used]; return(ndpi_serialization_type) (type & 0xf); } @@ -1605,13 +1904,13 @@ ndpi_serialization_type ndpi_deserialize_get_item_type(ndpi_deserializer *_deser /* ********************************** */ static inline int ndpi_deserialize_get_single_string_size(ndpi_private_deserializer *deserializer, u_int32_t offset) { - u_int32_t buff_diff = deserializer->buffer_size - offset; + u_int32_t buff_diff = deserializer->buffer.size - offset; u_int16_t expected, str_len; expected = sizeof(u_int16_t) /* len */; if(buff_diff < expected) return(-2); - str_len = ntohs(*((u_int16_t *) &deserializer->buffer[offset])); + str_len = ntohs(*((u_int16_t *) &deserializer->buffer.data[offset])); expected += str_len; if(buff_diff < expected) return(-2); @@ -1663,7 +1962,7 @@ static inline int ndpi_deserialize_get_single_size(ndpi_private_deserializer *de int ndpi_deserialize_next(ndpi_deserializer *_deserializer) { ndpi_private_deserializer *deserializer = (ndpi_private_deserializer *) _deserializer; - u_int32_t buff_diff = deserializer->buffer_size - deserializer->status.size_used; + u_int32_t buff_diff = deserializer->buffer.size - deserializer->status.buffer.size_used; ndpi_serialization_type kt, et; u_int16_t expected; int size; @@ -1673,20 +1972,20 @@ int ndpi_deserialize_next(ndpi_deserializer *_deserializer) { if(buff_diff < expected) return(-2); kt = ndpi_deserialize_get_key_subtype(deserializer); - size = ndpi_deserialize_get_single_size(deserializer, kt, deserializer->status.size_used + expected); + size = ndpi_deserialize_get_single_size(deserializer, kt, deserializer->status.buffer.size_used + expected); if(size < 0) return(-2); expected += size; et = ndpi_deserialize_get_value_subtype(deserializer); - size = ndpi_deserialize_get_single_size(deserializer, et, deserializer->status.size_used + expected); + size = ndpi_deserialize_get_single_size(deserializer, et, deserializer->status.buffer.size_used + expected); if(size < 0) return(-2); expected += size; - deserializer->status.size_used += expected; + deserializer->status.buffer.size_used += expected; return(0); } @@ -1696,7 +1995,7 @@ int ndpi_deserialize_next(ndpi_deserializer *_deserializer) { int ndpi_deserialize_key_uint32(ndpi_deserializer *_deserializer, u_int32_t *key) { ndpi_private_deserializer *deserializer = (ndpi_private_deserializer*)_deserializer; - u_int32_t offset, buff_diff = deserializer->buffer_size - deserializer->status.size_used; + u_int32_t offset, buff_diff = deserializer->buffer.size - deserializer->status.buffer.size_used; ndpi_serialization_type kt; u_int16_t expected; u_int16_t v16; @@ -1708,10 +2007,10 @@ int ndpi_deserialize_key_uint32(ndpi_deserializer *_deserializer, kt = ndpi_deserialize_get_key_subtype(deserializer); - size = ndpi_deserialize_get_single_size(deserializer, kt, deserializer->status.size_used + expected); + size = ndpi_deserialize_get_single_size(deserializer, kt, deserializer->status.buffer.size_used + expected); if(size < 0) return(-2); - offset = deserializer->status.size_used + expected; + offset = deserializer->status.buffer.size_used + expected; switch(kt) { case ndpi_serialization_uint32: @@ -1739,7 +2038,7 @@ int ndpi_deserialize_key_string(ndpi_deserializer *_deserializer, ndpi_string *key) { ndpi_private_deserializer *deserializer = (ndpi_private_deserializer*)_deserializer; ndpi_serialization_type kt; - u_int32_t buff_diff = deserializer->buffer_size - deserializer->status.size_used; + u_int32_t buff_diff = deserializer->buffer.size - deserializer->status.buffer.size_used; u_int16_t expected; int size; @@ -1748,10 +2047,10 @@ int ndpi_deserialize_key_string(ndpi_deserializer *_deserializer, kt = ndpi_deserialize_get_key_subtype(deserializer); - size = ndpi_deserialize_get_single_size(deserializer, kt, deserializer->status.size_used + expected); + size = ndpi_deserialize_get_single_size(deserializer, kt, deserializer->status.buffer.size_used + expected); if(size < 0) return(-2); - ndpi_deserialize_single_string(deserializer, deserializer->status.size_used + expected, key); + ndpi_deserialize_single_string(deserializer, deserializer->status.buffer.size_used + expected, key); return(0); } @@ -1762,7 +2061,7 @@ int ndpi_deserialize_value_uint32(ndpi_deserializer *_deserializer, u_int32_t *value) { ndpi_private_deserializer *deserializer = (ndpi_private_deserializer*)_deserializer; ndpi_serialization_type kt, et; - u_int32_t offset, buff_diff = deserializer->buffer_size - deserializer->status.size_used; + u_int32_t offset, buff_diff = deserializer->buffer.size - deserializer->status.buffer.size_used; u_int16_t v16; u_int8_t v8; u_int16_t expected; @@ -1772,16 +2071,16 @@ int ndpi_deserialize_value_uint32(ndpi_deserializer *_deserializer, if(buff_diff < expected) return(-2); kt = ndpi_deserialize_get_key_subtype(deserializer); - size = ndpi_deserialize_get_single_size(deserializer, kt, deserializer->status.size_used + expected); + size = ndpi_deserialize_get_single_size(deserializer, kt, deserializer->status.buffer.size_used + expected); if(size < 0) return(-2); expected += size; et = ndpi_deserialize_get_value_subtype(deserializer); - size = ndpi_deserialize_get_single_size(deserializer, et, deserializer->status.size_used + expected); + size = ndpi_deserialize_get_single_size(deserializer, et, deserializer->status.buffer.size_used + expected); if(size < 0) return(-2); - offset = deserializer->status.size_used + expected; + offset = deserializer->status.buffer.size_used + expected; switch(et) { case ndpi_serialization_uint32: @@ -1808,7 +2107,7 @@ int ndpi_deserialize_value_uint64(ndpi_deserializer *_deserializer, u_int64_t *value) { ndpi_private_deserializer *deserializer = (ndpi_private_deserializer*)_deserializer; ndpi_serialization_type kt, et; - u_int32_t buff_diff = deserializer->buffer_size - deserializer->status.size_used; + u_int32_t buff_diff = deserializer->buffer.size - deserializer->status.buffer.size_used; u_int32_t v32; u_int16_t expected; int size; @@ -1818,13 +2117,13 @@ int ndpi_deserialize_value_uint64(ndpi_deserializer *_deserializer, if(buff_diff < expected) return(-2); kt = ndpi_deserialize_get_key_subtype(deserializer); - size = ndpi_deserialize_get_single_size(deserializer, kt, deserializer->status.size_used + expected); + size = ndpi_deserialize_get_single_size(deserializer, kt, deserializer->status.buffer.size_used + expected); if(size < 0) return(-2); expected += size; et = ndpi_deserialize_get_value_subtype(deserializer); - size = ndpi_deserialize_get_single_size(deserializer, et, deserializer->status.size_used + expected); + size = ndpi_deserialize_get_single_size(deserializer, et, deserializer->status.buffer.size_used + expected); if(size < 0) return(-2); if(et != ndpi_serialization_uint64) { @@ -1834,7 +2133,7 @@ int ndpi_deserialize_value_uint64(ndpi_deserializer *_deserializer, return(rc); } - ndpi_deserialize_single_uint64(deserializer, deserializer->status.size_used + expected, value); + ndpi_deserialize_single_uint64(deserializer, deserializer->status.buffer.size_used + expected, value); return(0); } @@ -1845,7 +2144,7 @@ int ndpi_deserialize_value_int32(ndpi_deserializer *_deserializer, int32_t *value) { ndpi_private_deserializer *deserializer = (ndpi_private_deserializer*)_deserializer; ndpi_serialization_type kt, et; - u_int32_t offset, buff_diff = deserializer->buffer_size - deserializer->status.size_used; + u_int32_t offset, buff_diff = deserializer->buffer.size - deserializer->status.buffer.size_used; int16_t v16; int8_t v8; u_int16_t expected; @@ -1855,16 +2154,16 @@ int ndpi_deserialize_value_int32(ndpi_deserializer *_deserializer, if(buff_diff < expected) return(-2); kt = ndpi_deserialize_get_key_subtype(deserializer); - size = ndpi_deserialize_get_single_size(deserializer, kt, deserializer->status.size_used + expected); + size = ndpi_deserialize_get_single_size(deserializer, kt, deserializer->status.buffer.size_used + expected); if(size < 0) return(-2); expected += size; et = ndpi_deserialize_get_value_subtype(deserializer); - size = ndpi_deserialize_get_single_size(deserializer, et, deserializer->status.size_used + expected); + size = ndpi_deserialize_get_single_size(deserializer, et, deserializer->status.buffer.size_used + expected); if(size < 0) return(-2); - offset = deserializer->status.size_used + expected; + offset = deserializer->status.buffer.size_used + expected; switch(et) { case ndpi_serialization_int32: @@ -1891,7 +2190,7 @@ int ndpi_deserialize_value_int64(ndpi_deserializer *_deserializer, int64_t *value) { ndpi_private_deserializer *deserializer = (ndpi_private_deserializer*)_deserializer; ndpi_serialization_type kt, et; - u_int32_t buff_diff = deserializer->buffer_size - deserializer->status.size_used; + u_int32_t buff_diff = deserializer->buffer.size - deserializer->status.buffer.size_used; int32_t v32; u_int16_t expected; int size; @@ -1901,13 +2200,13 @@ int ndpi_deserialize_value_int64(ndpi_deserializer *_deserializer, if(buff_diff < expected) return(-2); kt = ndpi_deserialize_get_key_subtype(deserializer); - size = ndpi_deserialize_get_single_size(deserializer, kt, deserializer->status.size_used + expected); + size = ndpi_deserialize_get_single_size(deserializer, kt, deserializer->status.buffer.size_used + expected); if(size < 0) return(-2); expected += size; et = ndpi_deserialize_get_value_subtype(deserializer); - size = ndpi_deserialize_get_single_size(deserializer, et, deserializer->status.size_used + expected); + size = ndpi_deserialize_get_single_size(deserializer, et, deserializer->status.buffer.size_used + expected); if(size < 0) return(-2); if(et != ndpi_serialization_int64) { @@ -1917,7 +2216,7 @@ int ndpi_deserialize_value_int64(ndpi_deserializer *_deserializer, return(rc); } - ndpi_deserialize_single_int64(deserializer, deserializer->status.size_used + expected, value); + ndpi_deserialize_single_int64(deserializer, deserializer->status.buffer.size_used + expected, value); return(0); } @@ -1928,7 +2227,7 @@ int ndpi_deserialize_value_float(ndpi_deserializer *_deserializer, float *value) { ndpi_private_deserializer *deserializer = (ndpi_private_deserializer*)_deserializer; ndpi_serialization_type kt, et; - u_int32_t buff_diff = deserializer->buffer_size - deserializer->status.size_used; + u_int32_t buff_diff = deserializer->buffer.size - deserializer->status.buffer.size_used; u_int16_t expected; int size; @@ -1936,19 +2235,19 @@ int ndpi_deserialize_value_float(ndpi_deserializer *_deserializer, if(buff_diff < expected) return(-2); kt = ndpi_deserialize_get_key_subtype(deserializer); - size = ndpi_deserialize_get_single_size(deserializer, kt, deserializer->status.size_used + expected); + size = ndpi_deserialize_get_single_size(deserializer, kt, deserializer->status.buffer.size_used + expected); if(size < 0) return(-2); expected += size; et = ndpi_deserialize_get_value_subtype(deserializer); - size = ndpi_deserialize_get_single_size(deserializer, et, deserializer->status.size_used + expected); + size = ndpi_deserialize_get_single_size(deserializer, et, deserializer->status.buffer.size_used + expected); if(size < 0) return(-2); if(et != ndpi_serialization_float) return(-1); - ndpi_deserialize_single_float(deserializer, deserializer->status.size_used + expected, value); + ndpi_deserialize_single_float(deserializer, deserializer->status.buffer.size_used + expected, value); return(0); } @@ -1959,7 +2258,7 @@ int ndpi_deserialize_value_string(ndpi_deserializer *_deserializer, ndpi_string *value) { ndpi_private_deserializer *deserializer = (ndpi_private_deserializer*)_deserializer; ndpi_serialization_type kt, et; - u_int32_t buff_diff = deserializer->buffer_size - deserializer->status.size_used; + u_int32_t buff_diff = deserializer->buffer.size - deserializer->status.buffer.size_used; u_int16_t expected; int size; @@ -1967,19 +2266,19 @@ int ndpi_deserialize_value_string(ndpi_deserializer *_deserializer, if(buff_diff < expected) return(-2); kt = ndpi_deserialize_get_key_subtype(deserializer); - size = ndpi_deserialize_get_single_size(deserializer, kt, deserializer->status.size_used + expected); + size = ndpi_deserialize_get_single_size(deserializer, kt, deserializer->status.buffer.size_used + expected); if(size < 0) return(-2); expected += size; et = ndpi_deserialize_get_value_subtype(deserializer); - size = ndpi_deserialize_get_single_size(deserializer, et, deserializer->status.size_used + expected); + size = ndpi_deserialize_get_single_size(deserializer, et, deserializer->status.buffer.size_used + expected); if(size < 0) return(-2); if(et != ndpi_serialization_string) return(-1); - ndpi_deserialize_single_string(deserializer, deserializer->status.size_used + expected, value); + ndpi_deserialize_single_string(deserializer, deserializer->status.buffer.size_used + expected, value); return(0); } @@ -1990,8 +2289,8 @@ int ndpi_deserialize_value_string(ndpi_deserializer *_deserializer, int ndpi_deserialize_clone_item(ndpi_deserializer *_deserializer, ndpi_serializer *_serializer) { ndpi_private_deserializer *deserializer = (ndpi_private_deserializer *) _deserializer; ndpi_private_serializer *serializer = (ndpi_private_serializer*)_serializer; - u_int32_t src_buff_diff = deserializer->buffer_size - deserializer->status.size_used; - u_int32_t dst_buff_diff = serializer->buffer_size - serializer->status.size_used; + u_int32_t src_buff_diff = deserializer->buffer.size - deserializer->status.buffer.size_used; + u_int32_t dst_buff_diff = serializer->buffer.size - serializer->status.buffer.size_used; ndpi_serialization_type kt, et; u_int16_t expected; int size; @@ -2004,30 +2303,30 @@ int ndpi_deserialize_clone_item(ndpi_deserializer *_deserializer, ndpi_serialize if(src_buff_diff < expected) return(-2); kt = ndpi_deserialize_get_key_subtype(deserializer); - size = ndpi_deserialize_get_single_size(deserializer, kt, deserializer->status.size_used + expected); + size = ndpi_deserialize_get_single_size(deserializer, kt, deserializer->status.buffer.size_used + expected); if(size < 0) return(-2); expected += size; et = ndpi_deserialize_get_value_subtype(deserializer); - size = ndpi_deserialize_get_single_size(deserializer, et, deserializer->status.size_used + expected); + size = ndpi_deserialize_get_single_size(deserializer, et, deserializer->status.buffer.size_used + expected); if(size < 0) return(-2); expected += size; if(dst_buff_diff < expected) { - if(ndpi_extend_serializer_buffer(_serializer, expected - dst_buff_diff) < 0) + if(ndpi_extend_serializer_buffer(&serializer->buffer, expected - dst_buff_diff) < 0) return(-1); - dst_buff_diff = serializer->buffer_size - serializer->status.size_used; + dst_buff_diff = serializer->buffer.size - serializer->status.buffer.size_used; } - memcpy(&serializer->buffer[serializer->status.size_used], - &deserializer->buffer[deserializer->status.size_used], + memcpy(&serializer->buffer.data[serializer->status.buffer.size_used], + &deserializer->buffer.data[deserializer->status.buffer.size_used], expected); - serializer->status.size_used += expected; + serializer->status.buffer.size_used += expected; return(0); } diff --git a/src/lib/ndpi_utils.c b/src/lib/ndpi_utils.c index 4a30b1954..00fb47dad 100644 --- a/src/lib/ndpi_utils.c +++ b/src/lib/ndpi_utils.c @@ -902,63 +902,44 @@ char* ndpi_base64_encode(unsigned char const* bytes_to_encode, size_t in_len) { } /* ********************************** */ -/* ********************************** */ -int ndpi_flow2json(struct ndpi_detection_module_struct *ndpi_struct, - struct ndpi_flow_struct *flow, - u_int8_t ip_version, - u_int8_t l4_protocol, u_int16_t vlan_id, - u_int32_t src_v4, u_int32_t dst_v4, - struct ndpi_in6_addr *src_v6, struct ndpi_in6_addr *dst_v6, - u_int16_t src_port, u_int16_t dst_port, - ndpi_protocol l7_protocol, - ndpi_serializer *serializer) { - char buf[64], src_name[32], dst_name[32]; - - if(ndpi_init_serializer(serializer, ndpi_serialization_format_json) == -1) - return(-1); +void ndpi_serialize_risk(ndpi_serializer *serializer, + struct ndpi_flow_struct *flow) { + if(flow->risk != 0) { + u_int32_t i; - if(ip_version == 4) { - inet_ntop(AF_INET, &src_v4, src_name, sizeof(src_name)); - inet_ntop(AF_INET, &dst_v4, dst_name, sizeof(dst_name)); - } else { - inet_ntop(AF_INET6, src_v6, src_name, sizeof(src_name)); - inet_ntop(AF_INET6, dst_v6, dst_name, sizeof(dst_name)); - /* For consistency across platforms replace :0: with :: */ - ndpi_patchIPv6Address(src_name), ndpi_patchIPv6Address(dst_name); + ndpi_serialize_start_of_block(serializer, "flow_risk"); + + for(i = 0; i < NDPI_MAX_RISK; i++) { + ndpi_risk_enum r = (ndpi_risk_enum)i; + + if(NDPI_ISSET_BIT(flow->risk, r)) + ndpi_serialize_uint32_string(serializer, i, ndpi_risk2str(r)); + } + + ndpi_serialize_end_of_block(serializer); } +} - ndpi_serialize_string_string(serializer, "src_ip", src_name); - ndpi_serialize_string_string(serializer, "dest_ip", dst_name); - if(src_port) ndpi_serialize_string_uint32(serializer, "src_port", src_port); - if(dst_port) ndpi_serialize_string_uint32(serializer, "dst_port", dst_port); - - switch(l4_protocol) { - case IPPROTO_TCP: - ndpi_serialize_string_string(serializer, "proto", "TCP"); - break; - - case IPPROTO_UDP: - ndpi_serialize_string_string(serializer, "proto", "UDP"); - break; - - case IPPROTO_ICMP: - ndpi_serialize_string_string(serializer, "proto", "ICMP"); - break; +/* ********************************** */ +/* ********************************** */ - default: - ndpi_serialize_string_uint32(serializer, "proto", l4_protocol); - break; - } +/* NOTE: serializer must have been already initialized */ +int ndpi_dpi2json(struct ndpi_detection_module_struct *ndpi_struct, + struct ndpi_flow_struct *flow, + ndpi_protocol l7_protocol, + ndpi_serializer *serializer) { + char buf[64]; + if(flow == NULL) return(-1); + ndpi_serialize_start_of_block(serializer, "ndpi"); + ndpi_serialize_risk(serializer, flow); ndpi_serialize_string_string(serializer, "proto", ndpi_protocol2name(ndpi_struct, l7_protocol, buf, sizeof(buf))); if(l7_protocol.category != NDPI_PROTOCOL_CATEGORY_UNSPECIFIED) ndpi_serialize_string_string(serializer, "category", ndpi_category_get_name(ndpi_struct, l7_protocol.category)); ndpi_serialize_end_of_block(serializer); - if(flow == NULL) return(0); - switch(l7_protocol.master_protocol ? l7_protocol.master_protocol : l7_protocol.app_protocol) { case NDPI_PROTOCOL_DHCP: ndpi_serialize_start_of_block(serializer, "dhcp"); @@ -1148,6 +1129,59 @@ int ndpi_flow2json(struct ndpi_detection_module_struct *ndpi_struct, /* ********************************** */ +/* NOTE: serializer is initialized by the function */ +int ndpi_flow2json(struct ndpi_detection_module_struct *ndpi_struct, + struct ndpi_flow_struct *flow, + u_int8_t ip_version, + u_int8_t l4_protocol, u_int16_t vlan_id, + u_int32_t src_v4, u_int32_t dst_v4, + struct ndpi_in6_addr *src_v6, struct ndpi_in6_addr *dst_v6, + u_int16_t src_port, u_int16_t dst_port, + ndpi_protocol l7_protocol, + ndpi_serializer *serializer) { + char src_name[32], dst_name[32]; + + if(ndpi_init_serializer(serializer, ndpi_serialization_format_json) == -1) + return(-1); + + if(ip_version == 4) { + inet_ntop(AF_INET, &src_v4, src_name, sizeof(src_name)); + inet_ntop(AF_INET, &dst_v4, dst_name, sizeof(dst_name)); + } else { + inet_ntop(AF_INET6, src_v6, src_name, sizeof(src_name)); + inet_ntop(AF_INET6, dst_v6, dst_name, sizeof(dst_name)); + /* For consistency across platforms replace :0: with :: */ + ndpi_patchIPv6Address(src_name), ndpi_patchIPv6Address(dst_name); + } + + ndpi_serialize_string_string(serializer, "src_ip", src_name); + ndpi_serialize_string_string(serializer, "dest_ip", dst_name); + if(src_port) ndpi_serialize_string_uint32(serializer, "src_port", src_port); + if(dst_port) ndpi_serialize_string_uint32(serializer, "dst_port", dst_port); + + switch(l4_protocol) { + case IPPROTO_TCP: + ndpi_serialize_string_string(serializer, "proto", "TCP"); + break; + + case IPPROTO_UDP: + ndpi_serialize_string_string(serializer, "proto", "UDP"); + break; + + case IPPROTO_ICMP: + ndpi_serialize_string_string(serializer, "proto", "ICMP"); + break; + + default: + ndpi_serialize_string_uint32(serializer, "proto", l4_protocol); + break; + } + + return(ndpi_dpi2json(ndpi_struct, flow, l7_protocol, serializer)); +} + +/* ********************************** */ + const char* ndpi_tunnel2str(ndpi_packet_tunnel tt) { switch(tt) { case ndpi_no_tunnel: @@ -1457,6 +1491,9 @@ const char* ndpi_risk2str(ndpi_risk_enum risk) { case NDPI_HTTP_SUSPICIOUS_URL: return("HTTP Suspicious URL"); + + case NDPI_HTTP_SUSPICIOUS_HEADER: + return("HTTP Suspicious Header"); default: snprintf(buf, sizeof(buf), "%d", (int)risk); diff --git a/src/lib/protocols/bittorrent.c b/src/lib/protocols/bittorrent.c index f1c62d431..9f7620558 100644 --- a/src/lib/protocols/bittorrent.c +++ b/src/lib/protocols/bittorrent.c @@ -43,7 +43,7 @@ struct ndpi_utp_hdr { u_int16_t sequence_nr, ack_nr; }; -static u_int8_t is_utp_pkt(const u_int8_t *payload, u_int payload_len) { +static u_int8_t is_utpv1_pkt(const u_int8_t *payload, u_int payload_len) { struct ndpi_utp_hdr *h = (struct ndpi_utp_hdr*)payload; if(payload_len < sizeof(struct ndpi_utp_hdr)) return(0); @@ -52,6 +52,9 @@ static u_int8_t is_utp_pkt(const u_int8_t *payload, u_int payload_len) { if(h->next_extension > 2) return(0); if(ntohl(h->window_size) > 65565) return(0); + if((h->window_size == 0) && (payload_len != sizeof(struct ndpi_utp_hdr))) + return(0); + return(1); } @@ -433,14 +436,10 @@ void ndpi_search_bittorrent(struct ndpi_detection_module_struct *ndpi_struct, st u_int8_t v0_extension = packet->payload[17]; u_int8_t v0_flags = packet->payload[18]; - /* Check if this is protocol v1 */ - u_int8_t v1_version = packet->payload[0]; - u_int8_t v1_extension = packet->payload[1]; - u_int32_t v1_window_size = *((u_int32_t*)&packet->payload[12]); - - if(is_utp_pkt(packet->payload, packet->payload_packet_len)) + if(is_utpv1_pkt(packet->payload, packet->payload_packet_len)) { + bt_proto = ndpi_strnstr((const char *)&packet->payload[20], "BitTorrent protocol", packet->payload_packet_len-20); goto bittorrent_found; - else if((packet->payload[0]== 0x60) + } else if((packet->payload[0]== 0x60) && (packet->payload[1]== 0x0) && (packet->payload[2]== 0x0) && (packet->payload[3]== 0x0) @@ -448,14 +447,7 @@ void ndpi_search_bittorrent(struct ndpi_detection_module_struct *ndpi_struct, st /* Heuristic */ bt_proto = ndpi_strnstr((const char *)&packet->payload[20], "BitTorrent protocol", packet->payload_packet_len-20); goto bittorrent_found; - /* CSGO/DOTA conflict */ - } else if(flow->packet_counter > 8 && ((v1_version & 0x0f) == 1) - && ((v1_version >> 4) < 5 /* ST_NUM_STATES */) - && (v1_extension < 3 /* EXT_NUM_EXT */) - && (v1_window_size < 32768 /* 32k */) - ) { - bt_proto = ndpi_strnstr((const char *)&packet->payload[20], "BitTorrent protocol", packet->payload_packet_len-20); - goto bittorrent_found; + /* CSGO/DOTA conflict */ } else if((v0_flags < 6 /* ST_NUM_STATES */) && (v0_extension < 3 /* EXT_NUM_EXT */)) { u_int32_t ts = ntohl(*((u_int32_t*)&(packet->payload[4]))); u_int32_t now; diff --git a/src/lib/protocols/http.c b/src/lib/protocols/http.c index abd422007..7b83b91e7 100644 --- a/src/lib/protocols/http.c +++ b/src/lib/protocols/http.c @@ -28,20 +28,16 @@ #include "ndpi_api.h" #include <stdlib.h> -static const char* binary_file_mimes[] = { - "exe", - "vnd.ms-cab-compressed", - "vnd.microsoft.portable-executable" - "x-msdownload", - "x-dosexec", - NULL -}; +static const char* binary_file_mimes_e[] = { "exe", NULL }; +static const char* binary_file_mimes_v[] = { "vnd.ms-cab-compressed", "vnd.microsoft.portable-executable", NULL }; +static const char* binary_file_mimes_x[] = { "x-msdownload", "x-dosexec", NULL }; +#define ATTACHMENT_LEN 3 static const char* binary_file_ext[] = { - ".exe", - ".msi", - ".cab", - NULL + "exe", + "msi", + "cab", + NULL }; static void ndpi_search_http_tcp(struct ndpi_detection_module_struct *ndpi_struct, @@ -49,7 +45,7 @@ static void ndpi_search_http_tcp(struct ndpi_detection_module_struct *ndpi_struc /* *********************************************** */ -static void ndpi_analyze_content_signature(struct ndpi_flow_struct *flow) { +static void ndpi_analyze_content_signature(struct ndpi_flow_struct *flow) { if((flow->initial_binary_bytes_len >= 2) && (flow->initial_binary_bytes[0] == 0x4D) && (flow->initial_binary_bytes[1] == 0x5A)) NDPI_SET_BIT(flow->risk, NDPI_BINARY_APPLICATION_TRANSFER); /* Win executable */ else if((flow->initial_binary_bytes_len >= 4) && (flow->initial_binary_bytes[0] == 0x7F) && (flow->initial_binary_bytes[1] == 'E') @@ -65,7 +61,7 @@ static void ndpi_analyze_content_signature(struct ndpi_flow_struct *flow) { NDPI_SET_BIT(flow->risk, NDPI_BINARY_APPLICATION_TRANSFER); /* Unix script (e.g. #!/bin/sh) */ else if(flow->initial_binary_bytes_len >= 8) { u_int8_t exec_pattern[] = { 0x64, 0x65, 0x78, 0x0A, 0x30, 0x33, 0x35, 0x00 }; - + if(memcmp(flow->initial_binary_bytes, exec_pattern, 8) == 0) NDPI_SET_BIT(flow->risk, NDPI_BINARY_APPLICATION_TRANSFER); /* Dalvik Executable (Android) */ } @@ -85,7 +81,7 @@ static int ndpi_search_http_tcp_again(struct ndpi_detection_module_struct *ndpi_ && (flow->http.response_status_code != 0) ) { /* stop extra processing */ - + if(flow->initial_binary_bytes_len) ndpi_analyze_content_signature(flow); flow->extra_packets_func = NULL; /* We're good now */ return(0); @@ -106,47 +102,69 @@ static ndpi_protocol_category_t ndpi_http_check_content(struct ndpi_detection_mo u_int app_len = sizeof("application"); if(packet->content_line.len > app_len) { - const char *app = (const char *)&packet->content_line.ptr[app_len]; - u_int app_len_avail = packet->content_line.len-app_len; - - if(ndpi_strncasestr(app, "mpeg", app_len_avail) != NULL) { + const char *app = (const char *)&packet->content_line.ptr[app_len]; + u_int app_len_avail = packet->content_line.len-app_len; + + if(strncasecmp(app, "mpeg", app_len_avail) == 0) { flow->guessed_category = flow->category = NDPI_PROTOCOL_CATEGORY_STREAMING; return(flow->category); - } else { - for (int i = 0; binary_file_mimes[i] != NULL; i++) { - if (ndpi_strncasestr(app, binary_file_mimes[i], app_len_avail) != NULL) { - flow->guessed_category = flow->category = NDPI_PROTOCOL_CATEGORY_DOWNLOAD_FT; - NDPI_SET_BIT(flow->risk, NDPI_BINARY_APPLICATION_TRANSFER); - NDPI_LOG_INFO(ndpi_struct, "found executable HTTP transfer"); - return(flow->category); - } - } + } else if(app_len_avail > 3) { + const char** cmp_mimes = NULL; + + switch(app[0]) { + case 'e': cmp_mimes = binary_file_mimes_e; break; + case 'v': cmp_mimes = binary_file_mimes_v; break; + case 'x': cmp_mimes = binary_file_mimes_x; break; + } + + if(cmp_mimes != NULL) { + u_int8_t i; + + for(i = 0; cmp_mimes[i] != NULL; i++) { + if(strncasecmp(app, cmp_mimes[i], app_len_avail) == 0) { + flow->guessed_category = flow->category = NDPI_PROTOCOL_CATEGORY_DOWNLOAD_FT; + NDPI_SET_BIT(flow->risk, NDPI_BINARY_APPLICATION_TRANSFER); + NDPI_LOG_INFO(ndpi_struct, "found executable HTTP transfer"); + return(flow->category); + } + } + } } } /* check for attachment */ - if (packet->content_disposition_line.len > 0) { - uint8_t attachment_len = sizeof("attachment; filename"); - if (packet->content_disposition_line.len > attachment_len) { - uint8_t filename_len = packet->content_disposition_line.len - attachment_len; - for (int i = 0; binary_file_ext[i] != NULL; i++) { - if (ndpi_strncasestr((const char*)&packet->content_disposition_line.ptr[attachment_len], - binary_file_ext[i], filename_len)) { - flow->guessed_category = flow->category = NDPI_PROTOCOL_CATEGORY_DOWNLOAD_FT; - NDPI_SET_BIT(flow->risk, NDPI_BINARY_APPLICATION_TRANSFER); - NDPI_LOG_INFO(ndpi_struct, "found executable HTTP transfer"); - return(flow->category); - } + if (packet->content_disposition_line.len > 0) { + u_int8_t attachment_len = sizeof("attachment; filename"); + + if(packet->content_disposition_line.len > attachment_len) { + u_int8_t filename_len = packet->content_disposition_line.len - attachment_len; + + if(filename_len > ATTACHMENT_LEN) { + attachment_len += filename_len-ATTACHMENT_LEN-1; + + if((attachment_len+ATTACHMENT_LEN) <= packet->content_disposition_line.len) { + for(int i = 0; binary_file_ext[i] != NULL; i++) { + /* Use memcmp in case content-disposition contains binary data */ + if(memcmp((const char*)&packet->content_disposition_line.ptr[attachment_len], + binary_file_ext[i], ATTACHMENT_LEN) == 0) { + flow->guessed_category = flow->category = NDPI_PROTOCOL_CATEGORY_DOWNLOAD_FT; + NDPI_SET_BIT(flow->risk, NDPI_BINARY_APPLICATION_TRANSFER); + NDPI_LOG_INFO(ndpi_struct, "found executable HTTP transfer"); + return(flow->category); + } + } + } + } } } - } + switch(packet->content_line.ptr[0]) { case 'a': if(strncasecmp((const char *)packet->content_line.ptr, "audio", ndpi_min(packet->content_line.len, 5)) == 0) flow->guessed_category = flow->category = NDPI_PROTOCOL_CATEGORY_MEDIA; break; - + case 'v': if(strncasecmp((const char *)packet->content_line.ptr, "video", ndpi_min(packet->content_line.len, 5)) == 0) @@ -178,7 +196,6 @@ static void ndpi_int_http_add_connection(struct ndpi_detection_module_struct *nd if((flow->guessed_host_protocol_id == NDPI_PROTOCOL_UNKNOWN) || (http_protocol != NDPI_PROTOCOL_HTTP)) flow->guessed_host_protocol_id = http_protocol; - category = ndpi_http_check_content(ndpi_struct, flow); ndpi_int_reset_protocol(flow); ndpi_set_detected_protocol(ndpi_struct, flow, flow->guessed_host_protocol_id, NDPI_PROTOCOL_HTTP); @@ -186,7 +203,7 @@ static void ndpi_int_http_add_connection(struct ndpi_detection_module_struct *nd flow->check_extra_packets = 1; flow->max_extra_packets_to_check = 5; flow->extra_packets_func = ndpi_search_http_tcp_again; - flow->http_detected = 1, flow->guessed_category = flow->category = category; + flow->http_detected = 1; } /* ************************************************************* */ @@ -220,7 +237,7 @@ static void setHttpUserAgent(struct ndpi_detection_module_struct *ndpi_struct, * https://github.com/ua-parser/uap-core/blob/master/regexes.yaml */ snprintf((char*)flow->protos.http.detected_os, - sizeof(flow->protos.http.detected_os), "%s", ua); + sizeof(flow->protos.http.detected_os), "%s", ua); } /* ************************************************************* */ @@ -232,7 +249,7 @@ static void ndpi_http_parse_subprotocol(struct ndpi_detection_module_struct *ndp if(double_col) double_col[0] = '\0'; - ndpi_match_hostname_protocol(ndpi_struct, flow, NDPI_PROTOCOL_HTTP, + ndpi_match_hostname_protocol(ndpi_struct, flow, NDPI_PROTOCOL_HTTP, (char *)flow->host_server_name, strlen((const char *)flow->host_server_name)); } @@ -246,7 +263,7 @@ static void ndpi_check_user_agent(struct ndpi_detection_module_struct *ndpi_stru if((!ua) || (ua[0] == '\0')) return; // printf("[%s:%d] ==> '%s'\n", __FILE__, __LINE__, ua); - + if((strlen(ua) < 4) || (!strcmp(ua, "test")) || (!strcmp(ua, "<?")) @@ -262,13 +279,13 @@ static void ndpi_check_numeric_ip(struct ndpi_detection_module_struct *ndpi_stru char *ip, u_int ip_len) { char buf[22]; struct in_addr ip_addr; - + strncpy(buf, ip, ip_len); buf[ip_len] = '\0'; ip_addr.s_addr = inet_addr(buf); if(strcmp(inet_ntoa(ip_addr), buf) == 0) { - NDPI_SET_BIT(flow->risk, NDPI_HTTP_NUMERIC_IP_HOST); + NDPI_SET_BIT(flow->risk, NDPI_HTTP_NUMERIC_IP_HOST); } } @@ -301,9 +318,9 @@ static void check_content_type_and_change_protocol(struct ndpi_detection_module_ int len = packet->http_url_name.len + packet->host_line.len + 1; if(isdigit(packet->host_line.ptr[0]) - && (packet->host_line.len < 21)) + && (packet->host_line.len < 21)) ndpi_check_numeric_ip(ndpi_struct, flow, (char*)packet->host_line.ptr, packet->host_line.len); - + flow->http.url = ndpi_malloc(len); if(flow->http.url) { strncpy(flow->http.url, (char*)packet->host_line.ptr, packet->host_line.len); @@ -352,7 +369,7 @@ static void check_content_type_and_change_protocol(struct ndpi_detection_module_ strncpy(ua, (const char *)packet->user_agent_line.ptr, mlen); ua[mlen] = '\0'; - + if(strncmp(ua, "Mozilla", 7) == 0) { char *parent = strchr(ua, '('); @@ -517,7 +534,8 @@ static void check_content_type_and_change_protocol(struct ndpi_detection_module_ strncpy(flow->http.content_type, (char*)packet->content_line.ptr, packet->content_line.len); flow->http.content_type[packet->content_line.len] = '\0'; - } + + flow->guessed_category = flow->category = ndpi_http_check_content(ndpi_struct, flow);} } if(flow->http_detected) { @@ -595,6 +613,106 @@ static void http_bitmask_exclude_other(struct ndpi_flow_struct *flow) NDPI_ADD_PROTOCOL_TO_BITMASK(flow->excluded_protocol_bitmask, NDPI_PROTOCOL_XBOX); } +/* *********************************************************************************************** */ + +/* Trick to speed-up detection */ +static const char* suspicious_http_header_keys_A[] = { "Arch", NULL}; +static const char* suspicious_http_header_keys_C[] = { "Cores", NULL}; +static const char* suspicious_http_header_keys_M[] = { "Mem", NULL}; +static const char* suspicious_http_header_keys_O[] = { "Os", "Osname", "Osversion", NULL}; +static const char* suspicious_http_header_keys_R[] = { "Root", NULL}; +static const char* suspicious_http_header_keys_S[] = { "S", NULL}; +static const char* suspicious_http_header_keys_T[] = { "TLS_version", NULL}; +static const char* suspicious_http_header_keys_U[] = { "Uuid", NULL}; +static const char* suspicious_http_header_keys_X[] = { "X-Hire-Me", NULL}; + +static int is_a_suspicious_header(const char* suspicious_headers[], struct ndpi_int_one_line_struct packet_line){ + int i; + unsigned int header_len; + const u_int8_t* header_limit; + + if((header_limit = memchr(packet_line.ptr, ':', packet_line.len))) { + header_len = header_limit - packet_line.ptr; + for(i=0; suspicious_headers[i] != NULL; i++){ + if(!strncasecmp((const char*) packet_line.ptr, + suspicious_headers[i], header_len)) + return 1; + } + } + + return 0; +} + +/* *********************************************************************************************** */ + +static void ndpi_check_http_header(struct ndpi_detection_module_struct *ndpi_struct, + struct ndpi_flow_struct *flow) { + u_int32_t i; + struct ndpi_packet_struct *packet = &flow->packet; + + for(i=0; (i < packet->parsed_lines) + && (packet->line[i].ptr != NULL) + && (packet->line[i].len > 0); i++) { + switch(packet->line[i].ptr[0]){ + case 'A': + if(is_a_suspicious_header(suspicious_http_header_keys_A, packet->line[i])) { + NDPI_SET_BIT(flow->risk, NDPI_HTTP_SUSPICIOUS_HEADER); + return; + } + break; + case 'C': + if(is_a_suspicious_header(suspicious_http_header_keys_C, packet->line[i])) { + NDPI_SET_BIT(flow->risk, NDPI_HTTP_SUSPICIOUS_HEADER); + return; + } + break; + case 'M': + if(is_a_suspicious_header(suspicious_http_header_keys_M, packet->line[i])) { + NDPI_SET_BIT(flow->risk, NDPI_HTTP_SUSPICIOUS_HEADER); + return; + } + break; + case 'O': + if(is_a_suspicious_header(suspicious_http_header_keys_O, packet->line[i])) { + NDPI_SET_BIT(flow->risk, NDPI_HTTP_SUSPICIOUS_HEADER); + return; + } + break; + case 'R': + if(is_a_suspicious_header(suspicious_http_header_keys_R, packet->line[i])) { + NDPI_SET_BIT(flow->risk, NDPI_HTTP_SUSPICIOUS_HEADER); + return; + } + break; + case 'S': + if(is_a_suspicious_header(suspicious_http_header_keys_S, packet->line[i])) { + NDPI_SET_BIT(flow->risk, NDPI_HTTP_SUSPICIOUS_HEADER); + return; + } + break; + case 'T': + if(is_a_suspicious_header(suspicious_http_header_keys_T, packet->line[i])) { + NDPI_SET_BIT(flow->risk, NDPI_HTTP_SUSPICIOUS_HEADER); + return; + } + break; + case 'U': + if(is_a_suspicious_header(suspicious_http_header_keys_U, packet->line[i])) { + NDPI_SET_BIT(flow->risk, NDPI_HTTP_SUSPICIOUS_HEADER); + return; + } + break; + case 'X': + if(is_a_suspicious_header(suspicious_http_header_keys_X, packet->line[i])) { + NDPI_SET_BIT(flow->risk, NDPI_HTTP_SUSPICIOUS_HEADER); + return; + } + + break; + } + } +} + /*************************************************************************************************/ static void ndpi_check_http_tcp(struct ndpi_detection_module_struct *ndpi_struct, @@ -694,6 +812,7 @@ static void ndpi_check_http_tcp(struct ndpi_detection_module_struct *ndpi_struct "Filename HTTP found: %d, we look for line info..\n", filename_start); ndpi_parse_packet_line_info(ndpi_struct, flow); + ndpi_check_http_header(ndpi_struct, flow); if(packet->parsed_lines <= 1) { NDPI_LOG_DBG2(ndpi_struct, diff --git a/src/lib/protocols/kerberos.c b/src/lib/protocols/kerberos.c index ff16545f5..f4c1a175a 100644 --- a/src/lib/protocols/kerberos.c +++ b/src/lib/protocols/kerberos.c @@ -202,11 +202,13 @@ void ndpi_search_kerberos(struct ndpi_detection_module_struct *ndpi_struct, if(body_offset < packet->payload_packet_len) { - u_int16_t name_offset; - - name_offset = body_offset + 13; - for(i=0; i<20; i++) if(packet->payload[name_offset] != 0x1b) name_offset++; /* ASN.1 */ - + u_int16_t name_offset = body_offset + 13; + + for(i=0; (i<20) && (name_offset < packet->payload_packet_len); i++) { + if(packet->payload[name_offset] != 0x1b) + name_offset++; /* ASN.1 */ + } + #ifdef KERBEROS_DEBUG printf("name_offset=%u [%02X %02X] [byte 0 must be 0x1b]\n", name_offset, packet->payload[name_offset], packet->payload[name_offset+1]); #endif @@ -256,30 +258,38 @@ void ndpi_search_kerberos(struct ndpi_detection_module_struct *ndpi_struct, } else snprintf(flow->protos.kerberos.username, sizeof(flow->protos.kerberos.username), "%s", cname_str); - for(i=0; i<14; i++) if(packet->payload[realm_offset] != 0x1b) realm_offset++; /* ASN.1 */ + for(i=0; (i < 14) && (realm_offset < packet->payload_packet_len); i++) { + if(packet->payload[realm_offset] != 0x1b) + realm_offset++; /* ASN.1 */ + } + #ifdef KERBEROS_DEBUG - printf("realm_offset=%u [%02X %02X] [byte 0 must be 0x1b]\n", realm_offset, packet->payload[realm_offset], packet->payload[realm_offset+1]); + printf("realm_offset=%u [%02X %02X] [byte 0 must be 0x1b]\n", realm_offset, + packet->payload[realm_offset], packet->payload[realm_offset+1]); #endif + realm_offset += 1; //if(num_cname == 2) realm_offset++; - realm_len = packet->payload[realm_offset]; + if(realm_offset < packet->payload_packet_len) { + realm_len = packet->payload[realm_offset]; - if((realm_offset+realm_len) < packet->payload_packet_len) { - char realm_str[48]; + if((realm_offset+realm_len) < packet->payload_packet_len) { + char realm_str[48]; - if(realm_len > sizeof(realm_str)-1) - realm_len = sizeof(realm_str)-1; + if(realm_len > sizeof(realm_str)-1) + realm_len = sizeof(realm_str)-1; - realm_offset += 1; + realm_offset += 1; - strncpy(realm_str, (char*)&packet->payload[realm_offset], realm_len); - realm_str[realm_len] = '\0'; - for(i=0; i<realm_len; i++) realm_str[i] = tolower(realm_str[i]); + strncpy(realm_str, (char*)&packet->payload[realm_offset], realm_len); + realm_str[realm_len] = '\0'; + for(i=0; i<realm_len; i++) realm_str[i] = tolower(realm_str[i]); #ifdef KERBEROS_DEBUG - printf("[AS-REQ][Kerberos Realm][len: %u][%s]\n", realm_len, realm_str); + printf("[AS-REQ][Kerberos Realm][len: %u][%s]\n", realm_len, realm_str); #endif - snprintf(flow->protos.kerberos.domain, sizeof(flow->protos.kerberos.domain), "%s", realm_str); + snprintf(flow->protos.kerberos.domain, sizeof(flow->protos.kerberos.domain), "%s", realm_str); + } } } } diff --git a/src/lib/protocols/spotify.c b/src/lib/protocols/spotify.c index a180a1ea7..d66109016 100644 --- a/src/lib/protocols/spotify.c +++ b/src/lib/protocols/spotify.c @@ -47,7 +47,7 @@ static void ndpi_check_spotify(struct ndpi_detection_module_struct *ndpi_struct, if((packet->udp->source == spotify_port) && (packet->udp->dest == spotify_port)) { - if(payload_len > 2) { + if(payload_len >= 7) { if(memcmp(packet->payload, "SpotUdp", 7) == 0) { NDPI_LOG_INFO(ndpi_struct, "found spotify udp dissector\n"); ndpi_int_spotify_add_connection(ndpi_struct, flow, 0); diff --git a/src/lib/protocols/stun.c b/src/lib/protocols/stun.c index fd783d099..87d090daf 100644 --- a/src/lib/protocols/stun.c +++ b/src/lib/protocols/stun.c @@ -161,7 +161,7 @@ static ndpi_int_stun_t ndpi_int_check_stun(struct ndpi_detection_module_struct * /* This looks like an invalid packet */ if(flow->protos.stun_ssl.stun.num_udp_pkts > 0) { - flow->guessed_host_protocol_id = NDPI_PROTOCOL_WHATSAPP_CALL; + // flow->guessed_host_protocol_id = NDPI_PROTOCOL_WHATSAPP_CALL; return(NDPI_IS_STUN); } else return(NDPI_IS_NOT_STUN); diff --git a/src/lib/protocols/tls.c b/src/lib/protocols/tls.c index 62b2e3cf8..222fa480d 100644 --- a/src/lib/protocols/tls.c +++ b/src/lib/protocols/tls.c @@ -34,13 +34,13 @@ extern char *strptime(const char *s, const char *format, struct tm *tm); extern int processClientServerHello(struct ndpi_detection_module_struct *ndpi_struct, struct ndpi_flow_struct *flow); -// #define DEBUG_TLS_MEMORY 1 -// #define DEBUG_TLS 1 - +// #define DEBUG_TLS_MEMORY 1 +// #define DEBUG_TLS 1 // #define DEBUG_CERTIFICATE_HASH -/* #define DEBUG_FINGERPRINT 1 */ +/* #define DEBUG_FINGERPRINT 1 */ +/* #define DEBUG_ENCRYPTED_SNI 1 */ /* NOTE @@ -390,77 +390,83 @@ static void processCertificateElements(struct ndpi_detection_module_struct *ndpi i += 3 /* skip the initial patten 55 1D 11 */; i++; /* skip the first type, 0x04 == BIT STRING, and jump to it's length */ - i += (packet->payload[i] & 0x80) ? (packet->payload[i] & 0x7F) : 0; /* skip BIT STRING length */ - i += 2; /* skip the second type, 0x30 == SEQUENCE, and jump to it's length */ - i += (packet->payload[i] & 0x80) ? (packet->payload[i] & 0x7F) : 0; /* skip SEQUENCE length */ - i++; - - while(i < packet->payload_packet_len) { - if(packet->payload[i] == 0x82) { - if((i < (packet->payload_packet_len - 1)) - && ((i + packet->payload[i + 1] + 2) < packet->payload_packet_len)) { - u_int8_t len = packet->payload[i + 1]; - char dNSName[256]; - - i += 2; - - /* The check "len > sizeof(dNSName) - 1" will be always false. If we add it, - the compiler is smart enough to detect it and throws a warning */ - if(len == 0 /* Looks something went wrong */) - break; + if(i < packet->payload_packet_len) { + i += (packet->payload[i] & 0x80) ? (packet->payload[i] & 0x7F) : 0; /* skip BIT STRING length */ + if(i < packet->payload_packet_len) { + i += 2; /* skip the second type, 0x30 == SEQUENCE, and jump to it's length */ + if(i < packet->payload_packet_len) { + i += (packet->payload[i] & 0x80) ? (packet->payload[i] & 0x7F) : 0; /* skip SEQUENCE length */ + i++; + + while(i < packet->payload_packet_len) { + if(packet->payload[i] == 0x82) { + if((i < (packet->payload_packet_len - 1)) + && ((i + packet->payload[i + 1] + 2) < packet->payload_packet_len)) { + u_int8_t len = packet->payload[i + 1]; + char dNSName[256]; + + i += 2; + + /* The check "len > sizeof(dNSName) - 1" will be always false. If we add it, + the compiler is smart enough to detect it and throws a warning */ + if(len == 0 /* Looks something went wrong */) + break; - strncpy(dNSName, (const char*)&packet->payload[i], len); - dNSName[len] = '\0'; + strncpy(dNSName, (const char*)&packet->payload[i], len); + dNSName[len] = '\0'; - cleanupServerName(dNSName, len); + cleanupServerName(dNSName, len); #if DEBUG_TLS - printf("[TLS] dNSName %s [%s]\n", dNSName, flow->protos.stun_ssl.ssl.client_requested_server_name); + printf("[TLS] dNSName %s [%s]\n", dNSName, flow->protos.stun_ssl.ssl.client_requested_server_name); #endif - if(matched_name == 0) { - if((dNSName[0] == '*') && strstr(flow->protos.stun_ssl.ssl.client_requested_server_name, &dNSName[1])) - matched_name = 1; - else if(strcmp(flow->protos.stun_ssl.ssl.client_requested_server_name, dNSName) == 0) - matched_name = 1; - } + if(matched_name == 0) { + if((dNSName[0] == '*') && strstr(flow->protos.stun_ssl.ssl.client_requested_server_name, &dNSName[1])) + matched_name = 1; + else if(strcmp(flow->protos.stun_ssl.ssl.client_requested_server_name, dNSName) == 0) + matched_name = 1; + } - if(flow->protos.stun_ssl.ssl.server_names == NULL) - flow->protos.stun_ssl.ssl.server_names = ndpi_strdup(dNSName), - flow->protos.stun_ssl.ssl.server_names_len = strlen(dNSName); - else { - u_int16_t dNSName_len = strlen(dNSName); - u_int16_t newstr_len = flow->protos.stun_ssl.ssl.server_names_len + dNSName_len + 1; - char *newstr = (char*)ndpi_realloc(flow->protos.stun_ssl.ssl.server_names, - flow->protos.stun_ssl.ssl.server_names_len+1, newstr_len+1); - - if(newstr) { - flow->protos.stun_ssl.ssl.server_names = newstr; - flow->protos.stun_ssl.ssl.server_names[flow->protos.stun_ssl.ssl.server_names_len] = ','; - strncpy(&flow->protos.stun_ssl.ssl.server_names[flow->protos.stun_ssl.ssl.server_names_len+1], - dNSName, dNSName_len+1); - flow->protos.stun_ssl.ssl.server_names[newstr_len] = '\0'; - flow->protos.stun_ssl.ssl.server_names_len = newstr_len; - } - } + if(flow->protos.stun_ssl.ssl.server_names == NULL) + flow->protos.stun_ssl.ssl.server_names = ndpi_strdup(dNSName), + flow->protos.stun_ssl.ssl.server_names_len = strlen(dNSName); + else { + u_int16_t dNSName_len = strlen(dNSName); + u_int16_t newstr_len = flow->protos.stun_ssl.ssl.server_names_len + dNSName_len + 1; + char *newstr = (char*)ndpi_realloc(flow->protos.stun_ssl.ssl.server_names, + flow->protos.stun_ssl.ssl.server_names_len+1, newstr_len+1); + + if(newstr) { + flow->protos.stun_ssl.ssl.server_names = newstr; + flow->protos.stun_ssl.ssl.server_names[flow->protos.stun_ssl.ssl.server_names_len] = ','; + strncpy(&flow->protos.stun_ssl.ssl.server_names[flow->protos.stun_ssl.ssl.server_names_len+1], + dNSName, dNSName_len+1); + flow->protos.stun_ssl.ssl.server_names[newstr_len] = '\0'; + flow->protos.stun_ssl.ssl.server_names_len = newstr_len; + } + } - if(!flow->l4.tcp.tls.subprotocol_detected) - if(ndpi_match_hostname_protocol(ndpi_struct, flow, NDPI_PROTOCOL_TLS, dNSName, len)) - flow->l4.tcp.tls.subprotocol_detected = 1; + if(!flow->l4.tcp.tls.subprotocol_detected) + if(ndpi_match_hostname_protocol(ndpi_struct, flow, NDPI_PROTOCOL_TLS, dNSName, len)) + flow->l4.tcp.tls.subprotocol_detected = 1; - i += len; - } else { + i += len; + } else { #if DEBUG_TLS - printf("[TLS] Leftover %u bytes", packet->payload_packet_len - i); + printf("[TLS] Leftover %u bytes", packet->payload_packet_len - i); #endif - break; + break; + } + } else { + break; + } + } /* while */ + + if(!matched_name) + NDPI_SET_BIT(flow->risk, NDPI_TLS_CERTIFICATE_MISMATCH); /* Certificate mismatch */ } - } else { - break; } - } /* while */ - - if(!matched_name) - NDPI_SET_BIT(flow->risk, NDPI_TLS_CERTIFICATE_MISMATCH); /* Certificate mismatch */ + } } } @@ -1252,10 +1258,59 @@ int processClientServerHello(struct ndpi_detection_module_struct *ndpi_struct, if(flow->protos.stun_ssl.ssl.tls_supported_versions == NULL) flow->protos.stun_ssl.ssl.tls_supported_versions = ndpi_strdup(version_str); } - + } else if(extension_id == 65486 /* encrypted server name */) { + /* + - https://tools.ietf.org/html/draft-ietf-tls-esni-06 + - https://blog.cloudflare.com/encrypted-sni/ + */ + u_int16_t e_offset = offset+extension_offset; + u_int16_t initial_offset = e_offset; + u_int16_t e_sni_len, cipher_suite = ntohs(*((u_int16_t*)&packet->payload[e_offset])); + + flow->protos.stun_ssl.ssl.encrypted_sni.cipher_suite = cipher_suite; + + e_offset += 2; /* Cipher suite len */ + + /* Key Share Entry */ + e_offset += 2; /* Group */ + e_offset += ntohs(*((u_int16_t*)&packet->payload[e_offset])) + 2; /* Lenght */ + + if((e_offset+4) < packet->payload_packet_len) { + /* Record Digest */ + e_offset += ntohs(*((u_int16_t*)&packet->payload[e_offset])) + 2; /* Lenght */ + + if((e_offset+4) < packet->payload_packet_len) { + e_sni_len = ntohs(*((u_int16_t*)&packet->payload[e_offset])); + e_offset += 2; + + if((e_offset+e_sni_len-extension_len-initial_offset) >= 0) { +#ifdef DEBUG_ENCRYPTED_SNI + printf("Client SSL [Encrypted Server Name len: %u]\n", e_sni_len); +#endif + + if(flow->protos.stun_ssl.ssl.encrypted_sni.esni == NULL) { + flow->protos.stun_ssl.ssl.encrypted_sni.esni = (char*)ndpi_malloc(e_sni_len*2+1); + + if(flow->protos.stun_ssl.ssl.encrypted_sni.esni) { + u_int16_t i, off; + + for(i=e_offset, off=0; i<(e_offset+e_sni_len); i++) { + int rc = sprintf(&flow->protos.stun_ssl.ssl.encrypted_sni.esni[off], "%02X", packet->payload[i] & 0XFF); + + if(rc <= 0) { + flow->protos.stun_ssl.ssl.encrypted_sni.esni[off] = '\0'; + break; + } else + off += rc; + } + } + } + } + } + } } - extension_offset += extension_len; + extension_offset += extension_len; /* Move to the next extension */ #ifdef DEBUG_TLS printf("Client SSL [extension_offset/len: %u/%u]\n", extension_offset, extension_len); |