diff options
Diffstat (limited to 'examples/c-collectd/c-collectd.c')
| -rw-r--r-- | examples/c-collectd/c-collectd.c | 1536 |
1 files changed, 1124 insertions, 412 deletions
diff --git a/examples/c-collectd/c-collectd.c b/examples/c-collectd/c-collectd.c index 6bdf59aa3..1878a472e 100644 --- a/examples/c-collectd/c-collectd.c +++ b/examples/c-collectd/c-collectd.c @@ -5,7 +5,6 @@ #include <stdio.h> #include <stdlib.h> #include <string.h> -#include <syslog.h> #include <sys/epoll.h> #include <sys/timerfd.h> #include <unistd.h> @@ -13,198 +12,348 @@ #include <ndpi_typedefs.h> #include "nDPIsrvd.h" +#include "utils.h" #define DEFAULT_COLLECTD_EXEC_INST "nDPIsrvd" -#define ERROR_EVENT_ID_MAX 17 -//#define GENERATE_TIMESTAMP 1 - -#define LOG(flags, format, ...) \ - if (quiet == 0) \ +#define MAX_RISKS_PER_FLOW 8 +#define MAX_SEVERITIES_PER_FLOW 4 +#define LOGGER_EARLY(is_error, fmt, ...) \ + do \ { \ - fprintf(stderr, format, __VA_ARGS__); \ - fprintf(stderr, "%s", "\n"); \ - } \ - else \ - { \ - syslog(flags, format, __VA_ARGS__); \ - } + if (enable_console_logging != 0) \ + { \ + logger_early(is_error, fmt, __VA_ARGS__); \ + } \ + else \ + { \ + logger(is_error, fmt, __VA_ARGS__); \ + } \ + } while (0) +//#define GENERATE_TIMESTAMP 1 struct flow_user_data { nDPIsrvd_ull last_flow_src_l4_payload_len; nDPIsrvd_ull last_flow_dst_l4_payload_len; - nDPIsrvd_ull detected_risks; + uint8_t risks[MAX_RISKS_PER_FLOW]; + uint8_t severities[MAX_SEVERITIES_PER_FLOW]; + uint8_t category; + uint8_t breed; + uint8_t confidence; + // "fallthroughs" if we are not in sync with nDPI + uint8_t risk_ndpid_invalid : 1; + uint8_t category_ndpid_invalid : 1; + uint8_t breed_ndpid_invalid : 1; + uint8_t confidence_ndpid_invalid : 1; + // detection status + uint8_t new_seen : 1; + uint8_t is_detected : 1; + uint8_t is_guessed : 1; + uint8_t is_not_detected : 1; + // flow state + uint8_t is_info : 1; + uint8_t is_finished : 1; + // Layer3 / Layer4 + uint8_t is_ip4 : 1; + uint8_t is_ip6 : 1; + uint8_t is_other_l3 : 1; + uint8_t is_tcp : 1; + uint8_t is_udp : 1; + uint8_t is_icmp : 1; + uint8_t is_other_l4 : 1; }; static int main_thread_shutdown = 0; static int collectd_timerfd = -1; static pid_t collectd_pid; +static int enable_console_logging = 0; static char * serv_optarg = NULL; static char * collectd_hostname = NULL; static char * collectd_interval = NULL; static char * instance_name = NULL; static nDPIsrvd_ull collectd_interval_ull = 0uL; -static int quiet = 0; static struct { - uint64_t json_lines; - uint64_t json_bytes; - - uint64_t flow_new_count; - uint64_t flow_end_count; - uint64_t flow_idle_count; - uint64_t flow_update_count; - uint64_t flow_analyse_count; - uint64_t flow_guessed_count; - uint64_t flow_detected_count; - uint64_t flow_detection_update_count; - uint64_t flow_not_detected_count; - - uint64_t packet_count; - uint64_t packet_flow_count; - - uint64_t init_count; - uint64_t reconnect_count; - uint64_t shutdown_count; - uint64_t status_count; - - uint64_t error_count_sum; - uint64_t error_count[ERROR_EVENT_ID_MAX]; - uint64_t error_unknown_count; - - uint64_t flow_src_total_bytes; - uint64_t flow_dst_total_bytes; - uint64_t flow_risky_count; - - uint64_t flow_breed_safe_count; - uint64_t flow_breed_acceptable_count; - uint64_t flow_breed_fun_count; - uint64_t flow_breed_unsafe_count; - uint64_t flow_breed_potentially_dangerous_count; - uint64_t flow_breed_tracker_ads_count; - uint64_t flow_breed_dangerous_count; - uint64_t flow_breed_unrated_count; - uint64_t flow_breed_unknown_count; - - uint64_t flow_category_media_count; - uint64_t flow_category_vpn_count; - uint64_t flow_category_email_count; - uint64_t flow_category_data_transfer_count; - uint64_t flow_category_web_count; - uint64_t flow_category_social_network_count; - uint64_t flow_category_download_count; - uint64_t flow_category_game_count; - uint64_t flow_category_chat_count; - uint64_t flow_category_voip_count; - uint64_t flow_category_database_count; - uint64_t flow_category_remote_access_count; - uint64_t flow_category_cloud_count; - uint64_t flow_category_network_count; - uint64_t flow_category_collaborative_count; - uint64_t flow_category_rpc_count; - uint64_t flow_category_streaming_count; - uint64_t flow_category_system_count; - uint64_t flow_category_software_update_count; - uint64_t flow_category_music_count; - uint64_t flow_category_video_count; - uint64_t flow_category_shopping_count; - uint64_t flow_category_productivity_count; - uint64_t flow_category_file_sharing_count; - uint64_t flow_category_mining_count; - uint64_t flow_category_malware_count; - uint64_t flow_category_advertisment_count; - uint64_t flow_category_banned_site_count; - uint64_t flow_category_site_unavail_count; - uint64_t flow_category_allowed_site_count; - uint64_t flow_category_antimalware_count; - uint64_t flow_category_crypto_currency_count; - uint64_t flow_category_unknown_count; - - uint64_t flow_l3_ip4_count; - uint64_t flow_l3_ip6_count; - uint64_t flow_l3_other_count; - - uint64_t flow_l4_tcp_count; - uint64_t flow_l4_udp_count; - uint64_t flow_l4_icmp_count; - uint64_t flow_l4_other_count; - - nDPIsrvd_ull flow_risk_count[NDPI_MAX_RISK - 1]; - nDPIsrvd_ull flow_risk_unknown_count; + struct + { + uint64_t json_lines; + uint64_t json_bytes; + + uint64_t flow_new_count; + uint64_t flow_end_count; + uint64_t flow_idle_count; + uint64_t flow_update_count; + uint64_t flow_analyse_count; + uint64_t flow_guessed_count; + uint64_t flow_detected_count; + uint64_t flow_detection_update_count; + uint64_t flow_not_detected_count; + + uint64_t packet_count; + uint64_t packet_flow_count; + + uint64_t init_count; + uint64_t reconnect_count; + uint64_t shutdown_count; + uint64_t status_count; + + uint64_t error_unknown_datalink; + uint64_t error_unknown_l3_protocol; + uint64_t error_unsupported_datalink; + uint64_t error_packet_too_short; + uint64_t error_packet_type_unknown; + uint64_t error_packet_header_invalid; + uint64_t error_ip4_packet_too_short; + uint64_t error_ip4_size_smaller_than_header; + uint64_t error_ip4_l4_payload_detection; + uint64_t error_ip6_packet_too_short; + uint64_t error_ip6_size_smaller_than_header; + uint64_t error_ip6_l4_payload_detection; + uint64_t error_tcp_packet_too_short; + uint64_t error_udp_packet_too_short; + uint64_t error_capture_size_smaller_than_packet; + uint64_t error_max_flows_to_track; + uint64_t error_flow_memory_alloc; + + uint64_t flow_src_total_bytes; + uint64_t flow_dst_total_bytes; + uint64_t flow_risky_count; + } counters; + + struct + { + uint64_t flow_state_info; + uint64_t flow_state_finished; + + uint64_t flow_breed_safe_count; + uint64_t flow_breed_acceptable_count; + uint64_t flow_breed_fun_count; + uint64_t flow_breed_unsafe_count; + uint64_t flow_breed_potentially_dangerous_count; + uint64_t flow_breed_tracker_ads_count; + uint64_t flow_breed_dangerous_count; + uint64_t flow_breed_unrated_count; + uint64_t flow_breed_unknown_count; + + uint64_t flow_category_unspecified_count; + uint64_t flow_category_media_count; + uint64_t flow_category_vpn_count; + uint64_t flow_category_email_count; + uint64_t flow_category_data_transfer_count; + uint64_t flow_category_web_count; + uint64_t flow_category_social_network_count; + uint64_t flow_category_download_count; + uint64_t flow_category_game_count; + uint64_t flow_category_chat_count; + uint64_t flow_category_voip_count; + uint64_t flow_category_database_count; + uint64_t flow_category_remote_access_count; + uint64_t flow_category_cloud_count; + uint64_t flow_category_network_count; + uint64_t flow_category_collaborative_count; + uint64_t flow_category_rpc_count; + uint64_t flow_category_streaming_count; + uint64_t flow_category_system_count; + uint64_t flow_category_software_update_count; + uint64_t flow_category_music_count; + uint64_t flow_category_video_count; + uint64_t flow_category_shopping_count; + uint64_t flow_category_productivity_count; + uint64_t flow_category_file_sharing_count; + uint64_t flow_category_conn_check_count; + uint64_t flow_category_iot_scada_count; + uint64_t flow_category_virt_assistant_count; + uint64_t flow_category_cybersecurity_count; + uint64_t flow_category_adult_content_count; + uint64_t flow_category_mining_count; + uint64_t flow_category_malware_count; + uint64_t flow_category_advertisment_count; + uint64_t flow_category_banned_site_count; + uint64_t flow_category_site_unavail_count; + uint64_t flow_category_allowed_site_count; + uint64_t flow_category_antimalware_count; + uint64_t flow_category_crypto_currency_count; + uint64_t flow_category_gambling_count; + uint64_t flow_category_unknown_count; + + uint64_t flow_confidence_by_port; + uint64_t flow_confidence_dpi_partial; + uint64_t flow_confidence_dpi_partial_cache; + uint64_t flow_confidence_dpi_cache; + uint64_t flow_confidence_dpi; + uint64_t flow_confidence_nbpf; + uint64_t flow_confidence_by_ip; + uint64_t flow_confidence_dpi_aggressive; + uint64_t flow_confidence_unknown; + + uint64_t flow_severity_low; + uint64_t flow_severity_medium; + uint64_t flow_severity_high; + uint64_t flow_severity_severe; + uint64_t flow_severity_critical; + uint64_t flow_severity_emergency; + uint64_t flow_severity_unknown; + + uint64_t flow_l3_ip4_count; + uint64_t flow_l3_ip6_count; + uint64_t flow_l3_other_count; + + uint64_t flow_l4_tcp_count; + uint64_t flow_l4_udp_count; + uint64_t flow_l4_icmp_count; + uint64_t flow_l4_other_count; + + uint64_t flow_active_count; + uint64_t flow_detected_count; + uint64_t flow_guessed_count; + uint64_t flow_not_detected_count; + + nDPIsrvd_ull flow_risk_count[NDPI_MAX_RISK - 1]; + nDPIsrvd_ull flow_risk_unknown_count; + } gauges[2]; } collectd_statistics = {}; -struct json_stat_map +struct global_map { char const * const json_key; - uint64_t * const collectd_stat; + struct + { + uint64_t * const global_stat_inc; + uint64_t * const global_stat_dec; + }; }; -static struct json_stat_map const flow_event_map[] = {{"new", &collectd_statistics.flow_new_count}, - {"end", &collectd_statistics.flow_end_count}, - {"idle", &collectd_statistics.flow_idle_count}, - {"update", &collectd_statistics.flow_update_count}, - {"analyse", &collectd_statistics.flow_analyse_count}, - {"guessed", &collectd_statistics.flow_guessed_count}, - {"detected", &collectd_statistics.flow_detected_count}, - {"detection-update", - &collectd_statistics.flow_detection_update_count}, - {"not-detected", &collectd_statistics.flow_not_detected_count}}; - -static struct json_stat_map const packet_event_map[] = {{"packet", &collectd_statistics.packet_count}, - {"packet-flow", &collectd_statistics.packet_flow_count}}; - -static struct json_stat_map const daemon_event_map[] = {{"init", &collectd_statistics.init_count}, - {"reconnect", &collectd_statistics.reconnect_count}, - {"shutdown", &collectd_statistics.shutdown_count}, - {"status", &collectd_statistics.status_count}}; - -static struct json_stat_map const breeds_map[] = {{"Safe", &collectd_statistics.flow_breed_safe_count}, - {"Acceptable", &collectd_statistics.flow_breed_acceptable_count}, - {"Fun", &collectd_statistics.flow_breed_fun_count}, - {"Unsafe", &collectd_statistics.flow_breed_unsafe_count}, - {"Potentially Dangerous", - &collectd_statistics.flow_breed_potentially_dangerous_count}, - {"Tracker\\/Ads", &collectd_statistics.flow_breed_tracker_ads_count}, - {"Dangerous", &collectd_statistics.flow_breed_dangerous_count}, - {"Unrated", &collectd_statistics.flow_breed_unrated_count}, - {NULL, &collectd_statistics.flow_breed_unknown_count}}; - -static struct json_stat_map const categories_map[] = { - {"Media", &collectd_statistics.flow_category_media_count}, - {"VPN", &collectd_statistics.flow_category_vpn_count}, - {"Email", &collectd_statistics.flow_category_email_count}, - {"DataTransfer", &collectd_statistics.flow_category_data_transfer_count}, - {"Web", &collectd_statistics.flow_category_web_count}, - {"SocialNetwork", &collectd_statistics.flow_category_social_network_count}, - {"Download-FileTransfer-FileSharing", &collectd_statistics.flow_category_download_count}, - {"Game", &collectd_statistics.flow_category_game_count}, - {"Chat", &collectd_statistics.flow_category_chat_count}, - {"VoIP", &collectd_statistics.flow_category_voip_count}, - {"Database", &collectd_statistics.flow_category_database_count}, - {"RemoteAccess", &collectd_statistics.flow_category_remote_access_count}, - {"Cloud", &collectd_statistics.flow_category_cloud_count}, - {"Network", &collectd_statistics.flow_category_network_count}, - {"Collaborative", &collectd_statistics.flow_category_collaborative_count}, - {"RPC", &collectd_statistics.flow_category_rpc_count}, - {"Streaming", &collectd_statistics.flow_category_streaming_count}, - {"System", &collectd_statistics.flow_category_system_count}, - {"SoftwareUpdate", &collectd_statistics.flow_category_software_update_count}, - {"Music", &collectd_statistics.flow_category_music_count}, - {"Video", &collectd_statistics.flow_category_video_count}, - {"Shopping", &collectd_statistics.flow_category_shopping_count}, - {"Productivity", &collectd_statistics.flow_category_productivity_count}, - {"FileSharing", &collectd_statistics.flow_category_file_sharing_count}, - {"Mining", &collectd_statistics.flow_category_mining_count}, - {"Malware", &collectd_statistics.flow_category_malware_count}, - {"Advertisement", &collectd_statistics.flow_category_advertisment_count}, - {"Banned_Site", &collectd_statistics.flow_category_banned_site_count}, - {"Site_Unavailable", &collectd_statistics.flow_category_site_unavail_count}, - {"Allowed_Site", &collectd_statistics.flow_category_allowed_site_count}, - {"Antimalware", &collectd_statistics.flow_category_antimalware_count}, - {"Crypto_Currency", &collectd_statistics.flow_category_crypto_currency_count}, - {NULL, &collectd_statistics.flow_category_unknown_count}}; +#define COLLECTD_STATS_COUNTER_PTR(member) \ + { \ + .global_stat_inc = &(collectd_statistics.counters.member), NULL \ + } +#define COLLECTD_STATS_GAUGE_PTR(member) \ + { \ + .global_stat_inc = &(collectd_statistics.gauges[0].member), \ + .global_stat_dec = &(collectd_statistics.gauges[1].member) \ + } +#define COLLECTD_STATS_COUNTER_INC(member) (collectd_statistics.counters.member++) +#define COLLECTD_STATS_GAUGE_RES(member) (collectd_statistics.gauges[0].member--) +#define COLLECTD_STATS_GAUGE_INC(member) (collectd_statistics.gauges[0].member++) +#define COLLECTD_STATS_GAUGE_DEC(member) (collectd_statistics.gauges[1].member++) +#define COLLECTD_STATS_GAUGE_SUB(member) (collectd_statistics.gauges[0].member -= collectd_statistics.gauges[1].member) +#define COLLECTD_STATS_MAP_NOTNULL(map, index) (map[index - 1].global_stat_dec != NULL) +#define COLLECTD_STATS_MAP_DEC(map, index) ((*map[index - 1].global_stat_dec)++) + +static struct global_map const flow_event_map[] = {{"new", COLLECTD_STATS_COUNTER_PTR(flow_new_count)}, + {"end", COLLECTD_STATS_COUNTER_PTR(flow_end_count)}, + {"idle", COLLECTD_STATS_COUNTER_PTR(flow_idle_count)}, + {"update", COLLECTD_STATS_COUNTER_PTR(flow_update_count)}, + {"analyse", COLLECTD_STATS_COUNTER_PTR(flow_analyse_count)}, + {"guessed", COLLECTD_STATS_COUNTER_PTR(flow_guessed_count)}, + {"detected", COLLECTD_STATS_COUNTER_PTR(flow_detected_count)}, + {"detection-update", + COLLECTD_STATS_COUNTER_PTR(flow_detection_update_count)}, + {"not-detected", + COLLECTD_STATS_COUNTER_PTR(flow_not_detected_count)}}; + +static struct global_map const packet_event_map[] = {{"packet", COLLECTD_STATS_COUNTER_PTR(packet_count)}, + {"packet-flow", COLLECTD_STATS_COUNTER_PTR(packet_flow_count)}}; + +static struct global_map const daemon_event_map[] = {{"init", COLLECTD_STATS_COUNTER_PTR(init_count)}, + {"reconnect", COLLECTD_STATS_COUNTER_PTR(reconnect_count)}, + {"shutdown", COLLECTD_STATS_COUNTER_PTR(shutdown_count)}, + {"status", COLLECTD_STATS_COUNTER_PTR(status_count)}}; + +static struct global_map const error_event_map[] = { + {"Unknown datalink layer packet", COLLECTD_STATS_COUNTER_PTR(error_unknown_datalink)}, + {"Unknown L3 protocol", COLLECTD_STATS_COUNTER_PTR(error_unknown_l3_protocol)}, + {"Unsupported datalink layer", COLLECTD_STATS_COUNTER_PTR(error_unsupported_datalink)}, + {"Packet too short", COLLECTD_STATS_COUNTER_PTR(error_packet_too_short)}, + {"Unknown packet type", COLLECTD_STATS_COUNTER_PTR(error_packet_type_unknown)}, + {"Packet header invalid", COLLECTD_STATS_COUNTER_PTR(error_packet_header_invalid)}, + {"IP4 packet too short", COLLECTD_STATS_COUNTER_PTR(error_ip4_packet_too_short)}, + {"Packet smaller than IP4 header", COLLECTD_STATS_COUNTER_PTR(error_ip4_size_smaller_than_header)}, + {"nDPI IPv4\\/L4 payload detection failed", COLLECTD_STATS_COUNTER_PTR(error_ip4_l4_payload_detection)}, + {"IP6 packet too short", COLLECTD_STATS_COUNTER_PTR(error_ip6_packet_too_short)}, + {"Packet smaller than IP6 header", COLLECTD_STATS_COUNTER_PTR(error_ip6_size_smaller_than_header)}, + {"nDPI IPv6\\/L4 payload detection failed", COLLECTD_STATS_COUNTER_PTR(error_ip6_l4_payload_detection)}, + {"TCP packet smaller than expected", COLLECTD_STATS_COUNTER_PTR(error_tcp_packet_too_short)}, + {"UDP packet smaller than expected", COLLECTD_STATS_COUNTER_PTR(error_udp_packet_too_short)}, + {"Captured packet size is smaller than expected packet size", + COLLECTD_STATS_COUNTER_PTR(error_capture_size_smaller_than_packet)}, + {"Max flows to track reached", COLLECTD_STATS_COUNTER_PTR(error_max_flows_to_track)}, + {"Flow memory allocation failed", COLLECTD_STATS_COUNTER_PTR(error_flow_memory_alloc)}}; + +static struct global_map const breeds_map[] = {{"Safe", COLLECTD_STATS_GAUGE_PTR(flow_breed_safe_count)}, + {"Acceptable", COLLECTD_STATS_GAUGE_PTR(flow_breed_acceptable_count)}, + {"Fun", COLLECTD_STATS_GAUGE_PTR(flow_breed_fun_count)}, + {"Unsafe", COLLECTD_STATS_GAUGE_PTR(flow_breed_unsafe_count)}, + {"Potentially Dangerous", + COLLECTD_STATS_GAUGE_PTR(flow_breed_potentially_dangerous_count)}, + {"Tracker\\/Ads", + COLLECTD_STATS_GAUGE_PTR(flow_breed_tracker_ads_count)}, + {"Dangerous", COLLECTD_STATS_GAUGE_PTR(flow_breed_dangerous_count)}, + {"Unrated", COLLECTD_STATS_GAUGE_PTR(flow_breed_unrated_count)}, + {NULL, COLLECTD_STATS_GAUGE_PTR(flow_breed_unknown_count)}}; + +static struct global_map const categories_map[] = { + {"Unspecified", COLLECTD_STATS_GAUGE_PTR(flow_category_unspecified_count)}, + {"Media", COLLECTD_STATS_GAUGE_PTR(flow_category_media_count)}, + {"VPN", COLLECTD_STATS_GAUGE_PTR(flow_category_vpn_count)}, + {"Email", COLLECTD_STATS_GAUGE_PTR(flow_category_email_count)}, + {"DataTransfer", COLLECTD_STATS_GAUGE_PTR(flow_category_data_transfer_count)}, + {"Web", COLLECTD_STATS_GAUGE_PTR(flow_category_web_count)}, + {"SocialNetwork", COLLECTD_STATS_GAUGE_PTR(flow_category_social_network_count)}, + {"Download", COLLECTD_STATS_GAUGE_PTR(flow_category_download_count)}, + {"Game", COLLECTD_STATS_GAUGE_PTR(flow_category_game_count)}, + {"Chat", COLLECTD_STATS_GAUGE_PTR(flow_category_chat_count)}, + {"VoIP", COLLECTD_STATS_GAUGE_PTR(flow_category_voip_count)}, + {"Database", COLLECTD_STATS_GAUGE_PTR(flow_category_database_count)}, + {"RemoteAccess", COLLECTD_STATS_GAUGE_PTR(flow_category_remote_access_count)}, + {"Cloud", COLLECTD_STATS_GAUGE_PTR(flow_category_cloud_count)}, + {"Network", COLLECTD_STATS_GAUGE_PTR(flow_category_network_count)}, + {"Collaborative", COLLECTD_STATS_GAUGE_PTR(flow_category_collaborative_count)}, + {"RPC", COLLECTD_STATS_GAUGE_PTR(flow_category_rpc_count)}, + {"Streaming", COLLECTD_STATS_GAUGE_PTR(flow_category_streaming_count)}, + {"System", COLLECTD_STATS_GAUGE_PTR(flow_category_system_count)}, + {"SoftwareUpdate", COLLECTD_STATS_GAUGE_PTR(flow_category_software_update_count)}, + {"Music", COLLECTD_STATS_GAUGE_PTR(flow_category_music_count)}, + {"Video", COLLECTD_STATS_GAUGE_PTR(flow_category_video_count)}, + {"Shopping", COLLECTD_STATS_GAUGE_PTR(flow_category_shopping_count)}, + {"Productivity", COLLECTD_STATS_GAUGE_PTR(flow_category_productivity_count)}, + {"FileSharing", COLLECTD_STATS_GAUGE_PTR(flow_category_file_sharing_count)}, + {"ConnCheck", COLLECTD_STATS_GAUGE_PTR(flow_category_conn_check_count)}, + {"IoT-Scada", COLLECTD_STATS_GAUGE_PTR(flow_category_iot_scada_count)}, + {"VirtAssistant", COLLECTD_STATS_GAUGE_PTR(flow_category_virt_assistant_count)}, + {"Cybersecurity", COLLECTD_STATS_GAUGE_PTR(flow_category_cybersecurity_count)}, + {"AdultContent", COLLECTD_STATS_GAUGE_PTR(flow_category_adult_content_count)}, + {"Mining", COLLECTD_STATS_GAUGE_PTR(flow_category_mining_count)}, + {"Malware", COLLECTD_STATS_GAUGE_PTR(flow_category_malware_count)}, + {"Advertisement", COLLECTD_STATS_GAUGE_PTR(flow_category_advertisment_count)}, + {"Banned_Site", COLLECTD_STATS_GAUGE_PTR(flow_category_banned_site_count)}, + {"Site_Unavailable", COLLECTD_STATS_GAUGE_PTR(flow_category_site_unavail_count)}, + {"Allowed_Site", COLLECTD_STATS_GAUGE_PTR(flow_category_allowed_site_count)}, + {"Antimalware", COLLECTD_STATS_GAUGE_PTR(flow_category_antimalware_count)}, + {"Crypto_Currency", COLLECTD_STATS_GAUGE_PTR(flow_category_crypto_currency_count)}, + {"Gambling", COLLECTD_STATS_GAUGE_PTR(flow_category_gambling_count)}, + {NULL, COLLECTD_STATS_GAUGE_PTR(flow_category_unknown_count)}}; + +static struct global_map const confidence_map[] = { + {"Match by port", COLLECTD_STATS_GAUGE_PTR(flow_confidence_by_port)}, + {"DPI (partial)", COLLECTD_STATS_GAUGE_PTR(flow_confidence_dpi_partial)}, + {"DPI (partial cache)", COLLECTD_STATS_GAUGE_PTR(flow_confidence_dpi_partial_cache)}, + {"DPI (cache)", COLLECTD_STATS_GAUGE_PTR(flow_confidence_dpi_cache)}, + {"DPI", COLLECTD_STATS_GAUGE_PTR(flow_confidence_dpi)}, + {"nBPF", COLLECTD_STATS_GAUGE_PTR(flow_confidence_nbpf)}, + {"Match by IP", COLLECTD_STATS_GAUGE_PTR(flow_confidence_by_ip)}, + {"DPI (aggressive)", COLLECTD_STATS_GAUGE_PTR(flow_confidence_dpi_aggressive)}, + {NULL, COLLECTD_STATS_GAUGE_PTR(flow_confidence_unknown)}}; + +static struct global_map const severity_map[] = {{"Low", COLLECTD_STATS_GAUGE_PTR(flow_severity_low)}, + {"Medium", COLLECTD_STATS_GAUGE_PTR(flow_severity_medium)}, + {"High", COLLECTD_STATS_GAUGE_PTR(flow_severity_high)}, + {"Severe", COLLECTD_STATS_GAUGE_PTR(flow_severity_severe)}, + {"Critical", COLLECTD_STATS_GAUGE_PTR(flow_severity_critical)}, + {"Emergency", COLLECTD_STATS_GAUGE_PTR(flow_severity_emergency)}, + {NULL, COLLECTD_STATS_GAUGE_PTR(flow_severity_unknown)}}; #ifdef ENABLE_MEMORY_PROFILING void nDPIsrvd_memprof_log_alloc(size_t alloc_size) @@ -255,11 +404,11 @@ static int create_collectd_timer(void) static void sighandler(int signum) { - LOG(LOG_DAEMON | LOG_NOTICE, "Received SIGNAL %d", signum); + logger(0, "Received SIGNAL %d", signum); if (main_thread_shutdown == 0) { - LOG(LOG_DAEMON | LOG_NOTICE, "%s", "Shutting down .."); + logger(0, "%s", "Shutting down .."); main_thread_shutdown = 1; } } @@ -270,7 +419,8 @@ static int parse_options(int argc, char ** argv, struct nDPIsrvd_socket * const static char const usage[] = "Usage: %s " - "[-s host] [-c hostname] [-n collectd-instance-name] [-i interval] [-q]\n\n" + "[-l] [-s host] [-c hostname] [-n collectd-instance-name] [-i interval] [-q]\n\n" + "\t-l\tLog to console instead of syslog.\n" "\t-s\tDestination where nDPIsrvd is listening on.\n" "\t-c\tCollectd hostname.\n" "\t \tThis value defaults to the environment variable COLLECTD_HOSTNAME.\n" @@ -278,14 +428,15 @@ static int parse_options(int argc, char ** argv, struct nDPIsrvd_socket * const "\t \tDefaults to: " DEFAULT_COLLECTD_EXEC_INST "\n" "\t-i\tInterval between print statistics to stdout.\n" - "\t \tThis value defaults to the environment variable COLLECTD_INTERVAL.\n" - "\t-q\tDo not print anything except collectd statistics.\n" - "\t \tAutomatically enabled if environment variables mentioned above are set.\n"; + "\t \tThis value defaults to the environment variable COLLECTD_INTERVAL.\n"; - while ((opt = getopt(argc, argv, "hs:c:n:i:q")) != -1) + while ((opt = getopt(argc, argv, "hls:c:n:i:")) != -1) { switch (opt) { + case 'l': + enable_console_logging = 1; + break; case 's': free(serv_optarg); serv_optarg = strdup(optarg); @@ -302,11 +453,8 @@ static int parse_options(int argc, char ** argv, struct nDPIsrvd_socket * const free(collectd_interval); collectd_interval = strdup(optarg); break; - case 'q': - quiet = 1; - break; default: - LOG(LOG_DAEMON | LOG_ERR, usage, argv[0]); + fprintf(stderr, usage, argv[0]); return 1; } } @@ -323,6 +471,10 @@ static int parse_options(int argc, char ** argv, struct nDPIsrvd_socket * const { collectd_hostname = strdup("localhost"); } + else + { + enable_console_logging = 0; + } } if (instance_name == NULL) @@ -337,28 +489,34 @@ static int parse_options(int argc, char ** argv, struct nDPIsrvd_socket * const { collectd_interval = strdup("60"); } + else + { + enable_console_logging = 0; + } + } + + if (enable_console_logging != 0) + { + enable_console_logger(); } if (str_value_to_ull(collectd_interval, &collectd_interval_ull) != CONVERSION_OK) { - LOG(LOG_DAEMON | LOG_ERR, "Collectd interval `%s' is not a valid number", collectd_interval); + LOGGER_EARLY(1, "Collectd interval `%s' is not a valid number", collectd_interval); return 1; } if (nDPIsrvd_setup_address(&sock->address, serv_optarg) != 0) { - LOG(LOG_DAEMON | LOG_ERR, "Could not parse address `%s'", serv_optarg); + LOGGER_EARLY(1, "Could not parse address `%s'", serv_optarg); return 1; } if (optind < argc) { - LOG(LOG_DAEMON | LOG_ERR, "%s", "Unexpected argument after options"); - if (quiet == 0) - { - LOG(0, "%s", ""); - LOG(0, usage, argv[0]); - } + LOGGER_EARLY(1, "%s", "Unexpected argument after options"); + LOGGER_EARLY(1, "%s", ""); + LOGGER_EARLY(1, usage, argv[0]); return 1; } @@ -366,23 +524,45 @@ static int parse_options(int argc, char ** argv, struct nDPIsrvd_socket * const } #ifdef GENERATE_TIMESTAMP -#define COLLECTD_PUTVAL_PREFIX "PUTVAL \"%s/exec-%s/gauge-" -#define COLLECTD_PUTVAL_SUFFIX "\" interval=%llu %llu:%llu\n" -#define COLLECTD_PUTVAL_N(value) \ +#define COLLECTD_COUNTER_PREFIX "PUTVAL \"%s/exec-%s/counter-" +#define COLLECTD_COUNTER_SUFFIX "\" interval=%llu %llu:%llu\n" +#define COLLECTD_COUNTER_N(value) \ collectd_hostname, instance_name, #value, collectd_interval_ull, (unsigned long long int)now, \ (unsigned long long int)collectd_statistics.value -#define COLLECTD_PUTVAL_N2(name, value) \ +#define COLLECTD_COUNTER_N2(name, value) \ + collectd_hostname, instance_name, name, collectd_interval_ull, (unsigned long long int)now, \ + (unsigned long long int)collectd_statistics.value + +#define COLLECTD_GAUGE_PREFIX "PUTVAL \"%s/exec-%s/gauge-" +#define COLLECTD_GAUGE_SUFFIX "\" interval=%llu %llu:%llu\n" +#define COLLECTD_GAUGE_N(value) \ + collectd_hostname, instance_name, #value, collectd_interval_ull, (unsigned long long int)now, \ + (unsigned long long int)collectd_statistics.value +#define COLLECTD_GAUGE_N2(name, value) \ collectd_hostname, instance_name, name, collectd_interval_ull, (unsigned long long int)now, \ (unsigned long long int)collectd_statistics.value #else -#define COLLECTD_PUTVAL_PREFIX "PUTVAL \"%s/exec-%s/gauge-" -#define COLLECTD_PUTVAL_SUFFIX "\" interval=%llu N:%llu\n" -#define COLLECTD_PUTVAL_N(value) \ - collectd_hostname, instance_name, #value, collectd_interval_ull, (unsigned long long int)collectd_statistics.value -#define COLLECTD_PUTVAL_N2(name, value) \ - collectd_hostname, instance_name, name, collectd_interval_ull, (unsigned long long int)collectd_statistics.value +#define COLLECTD_COUNTER_PREFIX "PUTVAL \"%s/exec-%s/counter-" +#define COLLECTD_COUNTER_SUFFIX "\" interval=%llu N:%llu\n" +#define COLLECTD_COUNTER_N(value) \ + collectd_hostname, instance_name, #value, collectd_interval_ull, \ + (unsigned long long int)collectd_statistics.counters.value +#define COLLECTD_COUNTER_N2(name, value) \ + collectd_hostname, instance_name, name, collectd_interval_ull, \ + (unsigned long long int)collectd_statistics.counters.value + +#define COLLECTD_GAUGE_PREFIX "PUTVAL \"%s/exec-%s/gauge-" +#define COLLECTD_GAUGE_SUFFIX "\" interval=%llu N:%llu\n" +#define COLLECTD_GAUGE_N(value) \ + collectd_hostname, instance_name, #value, collectd_interval_ull, \ + (unsigned long long int)collectd_statistics.gauges[0].value +#define COLLECTD_GAUGE_N2(name, value) \ + collectd_hostname, instance_name, name, collectd_interval_ull, \ + (unsigned long long int)collectd_statistics.gauges[0].value #endif -#define COLLECTD_PUTVAL_N_FORMAT() COLLECTD_PUTVAL_PREFIX "%s" COLLECTD_PUTVAL_SUFFIX + +#define COLLECTD_COUNTER_N_FORMAT() COLLECTD_COUNTER_PREFIX "%s" COLLECTD_COUNTER_SUFFIX +#define COLLECTD_GAUGE_N_FORMAT() COLLECTD_GAUGE_PREFIX "%s" COLLECTD_GAUGE_SUFFIX static void print_collectd_exec_output(void) { size_t i; @@ -390,126 +570,275 @@ static void print_collectd_exec_output(void) time_t now = time(NULL); #endif - printf(COLLECTD_PUTVAL_N_FORMAT() COLLECTD_PUTVAL_N_FORMAT() COLLECTD_PUTVAL_N_FORMAT() COLLECTD_PUTVAL_N_FORMAT() - COLLECTD_PUTVAL_N_FORMAT() COLLECTD_PUTVAL_N_FORMAT() COLLECTD_PUTVAL_N_FORMAT() - COLLECTD_PUTVAL_N_FORMAT() COLLECTD_PUTVAL_N_FORMAT() COLLECTD_PUTVAL_N_FORMAT() - COLLECTD_PUTVAL_N_FORMAT() COLLECTD_PUTVAL_N_FORMAT() COLLECTD_PUTVAL_N_FORMAT() - COLLECTD_PUTVAL_N_FORMAT() COLLECTD_PUTVAL_N_FORMAT() COLLECTD_PUTVAL_N_FORMAT() - COLLECTD_PUTVAL_N_FORMAT() COLLECTD_PUTVAL_N_FORMAT() COLLECTD_PUTVAL_N_FORMAT() - COLLECTD_PUTVAL_N_FORMAT(), - - COLLECTD_PUTVAL_N(json_lines), - COLLECTD_PUTVAL_N(json_bytes), - COLLECTD_PUTVAL_N(flow_new_count), - COLLECTD_PUTVAL_N(flow_end_count), - COLLECTD_PUTVAL_N(flow_idle_count), - COLLECTD_PUTVAL_N(flow_update_count), - COLLECTD_PUTVAL_N(flow_analyse_count), - COLLECTD_PUTVAL_N(flow_guessed_count), - COLLECTD_PUTVAL_N(flow_detected_count), - COLLECTD_PUTVAL_N(flow_detection_update_count), - COLLECTD_PUTVAL_N(flow_not_detected_count), - COLLECTD_PUTVAL_N(flow_src_total_bytes), - COLLECTD_PUTVAL_N(flow_dst_total_bytes), - COLLECTD_PUTVAL_N(flow_risky_count), - COLLECTD_PUTVAL_N(packet_count), - COLLECTD_PUTVAL_N(packet_flow_count), - COLLECTD_PUTVAL_N(init_count), - COLLECTD_PUTVAL_N(reconnect_count), - COLLECTD_PUTVAL_N(shutdown_count), - COLLECTD_PUTVAL_N(status_count)); - - printf(COLLECTD_PUTVAL_N_FORMAT() COLLECTD_PUTVAL_N_FORMAT() COLLECTD_PUTVAL_N_FORMAT() COLLECTD_PUTVAL_N_FORMAT() - COLLECTD_PUTVAL_N_FORMAT() COLLECTD_PUTVAL_N_FORMAT() COLLECTD_PUTVAL_N_FORMAT() - COLLECTD_PUTVAL_N_FORMAT() COLLECTD_PUTVAL_N_FORMAT(), - - COLLECTD_PUTVAL_N(flow_breed_safe_count), - COLLECTD_PUTVAL_N(flow_breed_acceptable_count), - COLLECTD_PUTVAL_N(flow_breed_fun_count), - COLLECTD_PUTVAL_N(flow_breed_unsafe_count), - COLLECTD_PUTVAL_N(flow_breed_potentially_dangerous_count), - COLLECTD_PUTVAL_N(flow_breed_tracker_ads_count), - COLLECTD_PUTVAL_N(flow_breed_dangerous_count), - COLLECTD_PUTVAL_N(flow_breed_unrated_count), - COLLECTD_PUTVAL_N(flow_breed_unknown_count)); - - printf(COLLECTD_PUTVAL_N_FORMAT() COLLECTD_PUTVAL_N_FORMAT() COLLECTD_PUTVAL_N_FORMAT() COLLECTD_PUTVAL_N_FORMAT() - COLLECTD_PUTVAL_N_FORMAT() COLLECTD_PUTVAL_N_FORMAT() COLLECTD_PUTVAL_N_FORMAT() - COLLECTD_PUTVAL_N_FORMAT() COLLECTD_PUTVAL_N_FORMAT() COLLECTD_PUTVAL_N_FORMAT() - COLLECTD_PUTVAL_N_FORMAT() COLLECTD_PUTVAL_N_FORMAT() COLLECTD_PUTVAL_N_FORMAT() - COLLECTD_PUTVAL_N_FORMAT() COLLECTD_PUTVAL_N_FORMAT() COLLECTD_PUTVAL_N_FORMAT() - COLLECTD_PUTVAL_N_FORMAT() COLLECTD_PUTVAL_N_FORMAT() COLLECTD_PUTVAL_N_FORMAT() - COLLECTD_PUTVAL_N_FORMAT() COLLECTD_PUTVAL_N_FORMAT() COLLECTD_PUTVAL_N_FORMAT() - COLLECTD_PUTVAL_N_FORMAT() COLLECTD_PUTVAL_N_FORMAT() COLLECTD_PUTVAL_N_FORMAT() - COLLECTD_PUTVAL_N_FORMAT() COLLECTD_PUTVAL_N_FORMAT() - COLLECTD_PUTVAL_N_FORMAT() COLLECTD_PUTVAL_N_FORMAT() - COLLECTD_PUTVAL_N_FORMAT() COLLECTD_PUTVAL_N_FORMAT() - COLLECTD_PUTVAL_N_FORMAT() COLLECTD_PUTVAL_N_FORMAT(), - - COLLECTD_PUTVAL_N(flow_category_media_count), - COLLECTD_PUTVAL_N(flow_category_vpn_count), - COLLECTD_PUTVAL_N(flow_category_email_count), - COLLECTD_PUTVAL_N(flow_category_data_transfer_count), - COLLECTD_PUTVAL_N(flow_category_web_count), - COLLECTD_PUTVAL_N(flow_category_social_network_count), - COLLECTD_PUTVAL_N(flow_category_download_count), - COLLECTD_PUTVAL_N(flow_category_game_count), - COLLECTD_PUTVAL_N(flow_category_chat_count), - COLLECTD_PUTVAL_N(flow_category_voip_count), - COLLECTD_PUTVAL_N(flow_category_database_count), - COLLECTD_PUTVAL_N(flow_category_remote_access_count), - COLLECTD_PUTVAL_N(flow_category_cloud_count), - COLLECTD_PUTVAL_N(flow_category_network_count), - COLLECTD_PUTVAL_N(flow_category_collaborative_count), - COLLECTD_PUTVAL_N(flow_category_rpc_count), - COLLECTD_PUTVAL_N(flow_category_streaming_count), - COLLECTD_PUTVAL_N(flow_category_system_count), - COLLECTD_PUTVAL_N(flow_category_software_update_count), - COLLECTD_PUTVAL_N(flow_category_music_count), - COLLECTD_PUTVAL_N(flow_category_video_count), - COLLECTD_PUTVAL_N(flow_category_shopping_count), - COLLECTD_PUTVAL_N(flow_category_productivity_count), - COLLECTD_PUTVAL_N(flow_category_file_sharing_count), - COLLECTD_PUTVAL_N(flow_category_mining_count), - COLLECTD_PUTVAL_N(flow_category_malware_count), - COLLECTD_PUTVAL_N(flow_category_advertisment_count), - COLLECTD_PUTVAL_N(flow_category_banned_site_count), - COLLECTD_PUTVAL_N(flow_category_site_unavail_count), - COLLECTD_PUTVAL_N(flow_category_allowed_site_count), - COLLECTD_PUTVAL_N(flow_category_antimalware_count), - COLLECTD_PUTVAL_N(flow_category_crypto_currency_count), - COLLECTD_PUTVAL_N(flow_category_unknown_count)); - - printf(COLLECTD_PUTVAL_N_FORMAT() COLLECTD_PUTVAL_N_FORMAT() COLLECTD_PUTVAL_N_FORMAT() COLLECTD_PUTVAL_N_FORMAT() - COLLECTD_PUTVAL_N_FORMAT() COLLECTD_PUTVAL_N_FORMAT() COLLECTD_PUTVAL_N_FORMAT() - COLLECTD_PUTVAL_N_FORMAT() COLLECTD_PUTVAL_N_FORMAT() COLLECTD_PUTVAL_N_FORMAT(), - - COLLECTD_PUTVAL_N(flow_l3_ip4_count), - COLLECTD_PUTVAL_N(flow_l3_ip6_count), - COLLECTD_PUTVAL_N(flow_l3_other_count), - COLLECTD_PUTVAL_N(flow_l4_tcp_count), - COLLECTD_PUTVAL_N(flow_l4_udp_count), - COLLECTD_PUTVAL_N(flow_l4_icmp_count), - COLLECTD_PUTVAL_N(flow_l4_other_count), - COLLECTD_PUTVAL_N(flow_risk_unknown_count), - COLLECTD_PUTVAL_N(error_unknown_count), - COLLECTD_PUTVAL_N(error_count_sum)); - - for (i = 0; i < ERROR_EVENT_ID_MAX; ++i) - { - char gauge_name[BUFSIZ]; - snprintf(gauge_name, sizeof(gauge_name), "error_%zu_count", i); - printf(COLLECTD_PUTVAL_N_FORMAT(), COLLECTD_PUTVAL_N2(gauge_name, error_count[i])); - } + printf(COLLECTD_COUNTER_N_FORMAT() COLLECTD_COUNTER_N_FORMAT() COLLECTD_COUNTER_N_FORMAT() + COLLECTD_COUNTER_N_FORMAT() COLLECTD_COUNTER_N_FORMAT() COLLECTD_COUNTER_N_FORMAT() + COLLECTD_COUNTER_N_FORMAT() COLLECTD_COUNTER_N_FORMAT() COLLECTD_COUNTER_N_FORMAT() + COLLECTD_COUNTER_N_FORMAT() COLLECTD_COUNTER_N_FORMAT() COLLECTD_COUNTER_N_FORMAT() + COLLECTD_COUNTER_N_FORMAT() COLLECTD_COUNTER_N_FORMAT() COLLECTD_COUNTER_N_FORMAT() + COLLECTD_COUNTER_N_FORMAT() COLLECTD_COUNTER_N_FORMAT() COLLECTD_COUNTER_N_FORMAT() + COLLECTD_COUNTER_N_FORMAT() COLLECTD_COUNTER_N_FORMAT() COLLECTD_COUNTER_N_FORMAT() + COLLECTD_COUNTER_N_FORMAT() COLLECTD_COUNTER_N_FORMAT() + COLLECTD_COUNTER_N_FORMAT() COLLECTD_COUNTER_N_FORMAT() + COLLECTD_COUNTER_N_FORMAT() COLLECTD_COUNTER_N_FORMAT() + COLLECTD_COUNTER_N_FORMAT() COLLECTD_COUNTER_N_FORMAT() + COLLECTD_COUNTER_N_FORMAT() COLLECTD_COUNTER_N_FORMAT() + COLLECTD_COUNTER_N_FORMAT() COLLECTD_COUNTER_N_FORMAT() + COLLECTD_COUNTER_N_FORMAT() COLLECTD_COUNTER_N_FORMAT() + COLLECTD_COUNTER_N_FORMAT() + COLLECTD_COUNTER_N_FORMAT(), + + COLLECTD_COUNTER_N(json_lines), + COLLECTD_COUNTER_N(json_bytes), + COLLECTD_COUNTER_N(flow_new_count), + COLLECTD_COUNTER_N(flow_end_count), + COLLECTD_COUNTER_N(flow_idle_count), + COLLECTD_COUNTER_N(flow_update_count), + COLLECTD_COUNTER_N(flow_analyse_count), + COLLECTD_COUNTER_N(flow_guessed_count), + COLLECTD_COUNTER_N(flow_detected_count), + COLLECTD_COUNTER_N(flow_detection_update_count), + COLLECTD_COUNTER_N(flow_not_detected_count), + COLLECTD_COUNTER_N(flow_src_total_bytes), + COLLECTD_COUNTER_N(flow_dst_total_bytes), + COLLECTD_COUNTER_N(flow_risky_count), + COLLECTD_COUNTER_N(packet_count), + COLLECTD_COUNTER_N(packet_flow_count), + COLLECTD_COUNTER_N(init_count), + COLLECTD_COUNTER_N(reconnect_count), + COLLECTD_COUNTER_N(shutdown_count), + COLLECTD_COUNTER_N(status_count), + COLLECTD_COUNTER_N(error_unknown_datalink), + COLLECTD_COUNTER_N(error_unknown_l3_protocol), + COLLECTD_COUNTER_N(error_unsupported_datalink), + COLLECTD_COUNTER_N(error_packet_too_short), + COLLECTD_COUNTER_N(error_packet_type_unknown), + COLLECTD_COUNTER_N(error_packet_header_invalid), + COLLECTD_COUNTER_N(error_ip4_packet_too_short), + COLLECTD_COUNTER_N(error_ip4_size_smaller_than_header), + COLLECTD_COUNTER_N(error_ip4_l4_payload_detection), + COLLECTD_COUNTER_N(error_ip6_packet_too_short), + COLLECTD_COUNTER_N(error_ip6_size_smaller_than_header), + COLLECTD_COUNTER_N(error_ip6_l4_payload_detection), + COLLECTD_COUNTER_N(error_tcp_packet_too_short), + COLLECTD_COUNTER_N(error_udp_packet_too_short), + COLLECTD_COUNTER_N(error_capture_size_smaller_than_packet), + COLLECTD_COUNTER_N(error_max_flows_to_track), + COLLECTD_COUNTER_N(error_flow_memory_alloc)); + + printf(COLLECTD_GAUGE_N_FORMAT() COLLECTD_GAUGE_N_FORMAT() COLLECTD_GAUGE_N_FORMAT() COLLECTD_GAUGE_N_FORMAT() + COLLECTD_GAUGE_N_FORMAT() COLLECTD_GAUGE_N_FORMAT() COLLECTD_GAUGE_N_FORMAT() COLLECTD_GAUGE_N_FORMAT() + COLLECTD_GAUGE_N_FORMAT(), + + COLLECTD_GAUGE_N(flow_breed_safe_count), + COLLECTD_GAUGE_N(flow_breed_acceptable_count), + COLLECTD_GAUGE_N(flow_breed_fun_count), + COLLECTD_GAUGE_N(flow_breed_unsafe_count), + COLLECTD_GAUGE_N(flow_breed_potentially_dangerous_count), + COLLECTD_GAUGE_N(flow_breed_tracker_ads_count), + COLLECTD_GAUGE_N(flow_breed_dangerous_count), + COLLECTD_GAUGE_N(flow_breed_unrated_count), + COLLECTD_GAUGE_N(flow_breed_unknown_count)); + + printf(COLLECTD_GAUGE_N_FORMAT() COLLECTD_GAUGE_N_FORMAT() COLLECTD_GAUGE_N_FORMAT() COLLECTD_GAUGE_N_FORMAT() + COLLECTD_GAUGE_N_FORMAT() COLLECTD_GAUGE_N_FORMAT() COLLECTD_GAUGE_N_FORMAT() COLLECTD_GAUGE_N_FORMAT() + COLLECTD_GAUGE_N_FORMAT() COLLECTD_GAUGE_N_FORMAT() COLLECTD_GAUGE_N_FORMAT() + COLLECTD_GAUGE_N_FORMAT() COLLECTD_GAUGE_N_FORMAT() COLLECTD_GAUGE_N_FORMAT() + COLLECTD_GAUGE_N_FORMAT() COLLECTD_GAUGE_N_FORMAT() COLLECTD_GAUGE_N_FORMAT() + COLLECTD_GAUGE_N_FORMAT() COLLECTD_GAUGE_N_FORMAT() COLLECTD_GAUGE_N_FORMAT() + COLLECTD_GAUGE_N_FORMAT() COLLECTD_GAUGE_N_FORMAT() COLLECTD_GAUGE_N_FORMAT() + COLLECTD_GAUGE_N_FORMAT() COLLECTD_GAUGE_N_FORMAT() COLLECTD_GAUGE_N_FORMAT() + COLLECTD_GAUGE_N_FORMAT() COLLECTD_GAUGE_N_FORMAT() COLLECTD_GAUGE_N_FORMAT() + COLLECTD_GAUGE_N_FORMAT() COLLECTD_GAUGE_N_FORMAT() + COLLECTD_GAUGE_N_FORMAT() COLLECTD_GAUGE_N_FORMAT() + COLLECTD_GAUGE_N_FORMAT() COLLECTD_GAUGE_N_FORMAT() + COLLECTD_GAUGE_N_FORMAT() COLLECTD_GAUGE_N_FORMAT() + COLLECTD_GAUGE_N_FORMAT() COLLECTD_GAUGE_N_FORMAT() + COLLECTD_GAUGE_N_FORMAT(), + + COLLECTD_GAUGE_N(flow_category_unspecified_count), + COLLECTD_GAUGE_N(flow_category_media_count), + COLLECTD_GAUGE_N(flow_category_vpn_count), + COLLECTD_GAUGE_N(flow_category_email_count), + COLLECTD_GAUGE_N(flow_category_data_transfer_count), + COLLECTD_GAUGE_N(flow_category_web_count), + COLLECTD_GAUGE_N(flow_category_social_network_count), + COLLECTD_GAUGE_N(flow_category_download_count), + COLLECTD_GAUGE_N(flow_category_game_count), + COLLECTD_GAUGE_N(flow_category_chat_count), + COLLECTD_GAUGE_N(flow_category_voip_count), + COLLECTD_GAUGE_N(flow_category_database_count), + COLLECTD_GAUGE_N(flow_category_remote_access_count), + COLLECTD_GAUGE_N(flow_category_cloud_count), + COLLECTD_GAUGE_N(flow_category_network_count), + COLLECTD_GAUGE_N(flow_category_collaborative_count), + COLLECTD_GAUGE_N(flow_category_rpc_count), + COLLECTD_GAUGE_N(flow_category_streaming_count), + COLLECTD_GAUGE_N(flow_category_system_count), + COLLECTD_GAUGE_N(flow_category_software_update_count), + COLLECTD_GAUGE_N(flow_category_music_count), + COLLECTD_GAUGE_N(flow_category_video_count), + COLLECTD_GAUGE_N(flow_category_shopping_count), + COLLECTD_GAUGE_N(flow_category_productivity_count), + COLLECTD_GAUGE_N(flow_category_file_sharing_count), + COLLECTD_GAUGE_N(flow_category_conn_check_count), + COLLECTD_GAUGE_N(flow_category_iot_scada_count), + COLLECTD_GAUGE_N(flow_category_virt_assistant_count), + COLLECTD_GAUGE_N(flow_category_cybersecurity_count), + COLLECTD_GAUGE_N(flow_category_adult_content_count), + COLLECTD_GAUGE_N(flow_category_mining_count), + COLLECTD_GAUGE_N(flow_category_malware_count), + COLLECTD_GAUGE_N(flow_category_advertisment_count), + COLLECTD_GAUGE_N(flow_category_banned_site_count), + COLLECTD_GAUGE_N(flow_category_site_unavail_count), + COLLECTD_GAUGE_N(flow_category_allowed_site_count), + COLLECTD_GAUGE_N(flow_category_antimalware_count), + COLLECTD_GAUGE_N(flow_category_crypto_currency_count), + COLLECTD_GAUGE_N(flow_category_gambling_count), + COLLECTD_GAUGE_N(flow_category_unknown_count)); + + printf(COLLECTD_GAUGE_N_FORMAT() COLLECTD_GAUGE_N_FORMAT() COLLECTD_GAUGE_N_FORMAT() COLLECTD_GAUGE_N_FORMAT() + COLLECTD_GAUGE_N_FORMAT() COLLECTD_GAUGE_N_FORMAT() COLLECTD_GAUGE_N_FORMAT() COLLECTD_GAUGE_N_FORMAT() + COLLECTD_GAUGE_N_FORMAT(), + + COLLECTD_GAUGE_N(flow_confidence_by_port), + COLLECTD_GAUGE_N(flow_confidence_dpi_partial), + COLLECTD_GAUGE_N(flow_confidence_dpi_partial_cache), + COLLECTD_GAUGE_N(flow_confidence_dpi_cache), + COLLECTD_GAUGE_N(flow_confidence_dpi), + COLLECTD_GAUGE_N(flow_confidence_nbpf), + COLLECTD_GAUGE_N(flow_confidence_by_ip), + COLLECTD_GAUGE_N(flow_confidence_dpi_aggressive), + COLLECTD_GAUGE_N(flow_confidence_unknown)); + + printf(COLLECTD_GAUGE_N_FORMAT() COLLECTD_GAUGE_N_FORMAT() COLLECTD_GAUGE_N_FORMAT() COLLECTD_GAUGE_N_FORMAT() + COLLECTD_GAUGE_N_FORMAT() COLLECTD_GAUGE_N_FORMAT() COLLECTD_GAUGE_N_FORMAT(), + + COLLECTD_GAUGE_N(flow_severity_low), + COLLECTD_GAUGE_N(flow_severity_medium), + COLLECTD_GAUGE_N(flow_severity_high), + COLLECTD_GAUGE_N(flow_severity_severe), + COLLECTD_GAUGE_N(flow_severity_critical), + COLLECTD_GAUGE_N(flow_severity_emergency), + COLLECTD_GAUGE_N(flow_severity_unknown)); + + printf(COLLECTD_GAUGE_N_FORMAT() COLLECTD_GAUGE_N_FORMAT() COLLECTD_GAUGE_N_FORMAT() COLLECTD_GAUGE_N_FORMAT() + COLLECTD_GAUGE_N_FORMAT() COLLECTD_GAUGE_N_FORMAT() COLLECTD_GAUGE_N_FORMAT() COLLECTD_GAUGE_N_FORMAT(), + + COLLECTD_GAUGE_N(flow_l3_ip4_count), + COLLECTD_GAUGE_N(flow_l3_ip6_count), + COLLECTD_GAUGE_N(flow_l3_other_count), + COLLECTD_GAUGE_N(flow_l4_tcp_count), + COLLECTD_GAUGE_N(flow_l4_udp_count), + COLLECTD_GAUGE_N(flow_l4_icmp_count), + COLLECTD_GAUGE_N(flow_l4_other_count), + COLLECTD_GAUGE_N(flow_risk_unknown_count)); + + printf(COLLECTD_GAUGE_N_FORMAT() COLLECTD_GAUGE_N_FORMAT() COLLECTD_GAUGE_N_FORMAT() COLLECTD_GAUGE_N_FORMAT(), + COLLECTD_GAUGE_N(flow_active_count), + COLLECTD_GAUGE_N(flow_detected_count), + COLLECTD_GAUGE_N(flow_guessed_count), + COLLECTD_GAUGE_N(flow_not_detected_count)); for (i = 0; i < NDPI_MAX_RISK - 1; ++i) { char gauge_name[BUFSIZ]; snprintf(gauge_name, sizeof(gauge_name), "flow_risk_%zu_count", i + 1); - printf(COLLECTD_PUTVAL_N_FORMAT(), COLLECTD_PUTVAL_N2(gauge_name, flow_risk_count[i])); + printf(COLLECTD_GAUGE_N_FORMAT(), COLLECTD_GAUGE_N2(gauge_name, flow_risk_count[i])); + } + + COLLECTD_STATS_GAUGE_SUB(flow_state_info); + COLLECTD_STATS_GAUGE_SUB(flow_state_finished); + + COLLECTD_STATS_GAUGE_SUB(flow_breed_safe_count); + COLLECTD_STATS_GAUGE_SUB(flow_breed_acceptable_count); + COLLECTD_STATS_GAUGE_SUB(flow_breed_fun_count); + COLLECTD_STATS_GAUGE_SUB(flow_breed_unsafe_count); + COLLECTD_STATS_GAUGE_SUB(flow_breed_potentially_dangerous_count); + COLLECTD_STATS_GAUGE_SUB(flow_breed_tracker_ads_count); + COLLECTD_STATS_GAUGE_SUB(flow_breed_dangerous_count); + COLLECTD_STATS_GAUGE_SUB(flow_breed_unrated_count); + COLLECTD_STATS_GAUGE_SUB(flow_breed_unknown_count); + + COLLECTD_STATS_GAUGE_SUB(flow_category_unspecified_count); + COLLECTD_STATS_GAUGE_SUB(flow_category_media_count); + COLLECTD_STATS_GAUGE_SUB(flow_category_vpn_count); + COLLECTD_STATS_GAUGE_SUB(flow_category_email_count); + COLLECTD_STATS_GAUGE_SUB(flow_category_data_transfer_count); + COLLECTD_STATS_GAUGE_SUB(flow_category_web_count); + COLLECTD_STATS_GAUGE_SUB(flow_category_social_network_count); + COLLECTD_STATS_GAUGE_SUB(flow_category_download_count); + COLLECTD_STATS_GAUGE_SUB(flow_category_game_count); + COLLECTD_STATS_GAUGE_SUB(flow_category_chat_count); + COLLECTD_STATS_GAUGE_SUB(flow_category_voip_count); + COLLECTD_STATS_GAUGE_SUB(flow_category_database_count); + COLLECTD_STATS_GAUGE_SUB(flow_category_remote_access_count); + COLLECTD_STATS_GAUGE_SUB(flow_category_cloud_count); + COLLECTD_STATS_GAUGE_SUB(flow_category_network_count); + COLLECTD_STATS_GAUGE_SUB(flow_category_collaborative_count); + COLLECTD_STATS_GAUGE_SUB(flow_category_rpc_count); + COLLECTD_STATS_GAUGE_SUB(flow_category_streaming_count); + COLLECTD_STATS_GAUGE_SUB(flow_category_system_count); + COLLECTD_STATS_GAUGE_SUB(flow_category_software_update_count); + COLLECTD_STATS_GAUGE_SUB(flow_category_music_count); + COLLECTD_STATS_GAUGE_SUB(flow_category_video_count); + COLLECTD_STATS_GAUGE_SUB(flow_category_shopping_count); + COLLECTD_STATS_GAUGE_SUB(flow_category_productivity_count); + COLLECTD_STATS_GAUGE_SUB(flow_category_file_sharing_count); + COLLECTD_STATS_GAUGE_SUB(flow_category_conn_check_count); + COLLECTD_STATS_GAUGE_SUB(flow_category_iot_scada_count); + COLLECTD_STATS_GAUGE_SUB(flow_category_virt_assistant_count); + COLLECTD_STATS_GAUGE_SUB(flow_category_cybersecurity_count); + COLLECTD_STATS_GAUGE_SUB(flow_category_adult_content_count); + COLLECTD_STATS_GAUGE_SUB(flow_category_mining_count); + COLLECTD_STATS_GAUGE_SUB(flow_category_malware_count); + COLLECTD_STATS_GAUGE_SUB(flow_category_advertisment_count); + COLLECTD_STATS_GAUGE_SUB(flow_category_banned_site_count); + COLLECTD_STATS_GAUGE_SUB(flow_category_site_unavail_count); + COLLECTD_STATS_GAUGE_SUB(flow_category_allowed_site_count); + COLLECTD_STATS_GAUGE_SUB(flow_category_antimalware_count); + COLLECTD_STATS_GAUGE_SUB(flow_category_crypto_currency_count); + COLLECTD_STATS_GAUGE_SUB(flow_category_gambling_count); + COLLECTD_STATS_GAUGE_SUB(flow_category_unknown_count); + + COLLECTD_STATS_GAUGE_SUB(flow_confidence_by_port); + COLLECTD_STATS_GAUGE_SUB(flow_confidence_dpi_partial); + COLLECTD_STATS_GAUGE_SUB(flow_confidence_dpi_partial_cache); + COLLECTD_STATS_GAUGE_SUB(flow_confidence_dpi_cache); + COLLECTD_STATS_GAUGE_SUB(flow_confidence_dpi); + COLLECTD_STATS_GAUGE_SUB(flow_confidence_nbpf); + COLLECTD_STATS_GAUGE_SUB(flow_confidence_by_ip); + COLLECTD_STATS_GAUGE_SUB(flow_confidence_dpi_aggressive); + COLLECTD_STATS_GAUGE_SUB(flow_confidence_unknown); + + COLLECTD_STATS_GAUGE_SUB(flow_severity_low); + COLLECTD_STATS_GAUGE_SUB(flow_severity_medium); + COLLECTD_STATS_GAUGE_SUB(flow_severity_high); + COLLECTD_STATS_GAUGE_SUB(flow_severity_severe); + COLLECTD_STATS_GAUGE_SUB(flow_severity_critical); + COLLECTD_STATS_GAUGE_SUB(flow_severity_emergency); + COLLECTD_STATS_GAUGE_SUB(flow_severity_unknown); + + COLLECTD_STATS_GAUGE_SUB(flow_l3_ip4_count); + COLLECTD_STATS_GAUGE_SUB(flow_l3_ip6_count); + COLLECTD_STATS_GAUGE_SUB(flow_l3_other_count); + + COLLECTD_STATS_GAUGE_SUB(flow_l4_tcp_count); + COLLECTD_STATS_GAUGE_SUB(flow_l4_udp_count); + COLLECTD_STATS_GAUGE_SUB(flow_l4_icmp_count); + COLLECTD_STATS_GAUGE_SUB(flow_l4_other_count); + + COLLECTD_STATS_GAUGE_SUB(flow_active_count); + COLLECTD_STATS_GAUGE_SUB(flow_detected_count); + COLLECTD_STATS_GAUGE_SUB(flow_guessed_count); + COLLECTD_STATS_GAUGE_SUB(flow_not_detected_count); + + for (size_t i = 0; i < NDPI_MAX_RISK - 1; ++i) + { + COLLECTD_STATS_GAUGE_SUB(flow_risk_count[i]); } + COLLECTD_STATS_GAUGE_SUB(flow_risk_unknown_count); - memset(&collectd_statistics, 0, sizeof(collectd_statistics)); + memset(&collectd_statistics.gauges[1], 0, sizeof(collectd_statistics.gauges[1])); } static int mainloop(int epollfd, struct nDPIsrvd_socket * const sock) @@ -525,7 +854,7 @@ static int mainloop(int epollfd, struct nDPIsrvd_socket * const sock) { if (events[i].events & EPOLLERR) { - LOG(LOG_DAEMON | LOG_ERR, "Epoll event error: %s", (errno != 0 ? strerror(errno) : "EPOLLERR")); + logger(1, "Epoll event error: %s", (errno != 0 ? strerror(errno) : "EPOLLERR")); break; } @@ -539,19 +868,19 @@ static int mainloop(int epollfd, struct nDPIsrvd_socket * const sock) */ if (getppid() != collectd_pid) { - LOG(LOG_DAEMON | LOG_ERR, "Parent process %d exited. Nothing left to do here, bye.", collectd_pid); + logger(1, "Parent process %d exited. Nothing left to do here, bye.", collectd_pid); return 1; } errno = 0; if (read(collectd_timerfd, &expirations, sizeof(expirations)) != sizeof(expirations)) { - LOG(LOG_DAEMON | LOG_ERR, "Could not read timer expirations: %s", strerror(errno)); + logger(1, "Could not read timer expirations: %s", strerror(errno)); return 1; } if (set_collectd_timer() != 0) { - LOG(LOG_DAEMON | LOG_ERR, "Could not set timer: %s", strerror(errno)); + logger(1, "Could not set timer: %s", strerror(errno)); return 1; } @@ -563,14 +892,14 @@ static int mainloop(int epollfd, struct nDPIsrvd_socket * const sock) enum nDPIsrvd_read_return read_ret = nDPIsrvd_read(sock); if (read_ret != READ_OK) { - LOG(LOG_DAEMON | LOG_ERR, "nDPIsrvd read failed with: %s", nDPIsrvd_enum_to_string(read_ret)); + logger(1, "nDPIsrvd read failed with: %s", nDPIsrvd_enum_to_string(read_ret)); return 1; } enum nDPIsrvd_parse_return parse_ret = nDPIsrvd_parse_all(sock); if (parse_ret != PARSE_NEED_MORE_DATA) { - LOG(LOG_DAEMON | LOG_ERR, "nDPIsrvd parse failed with: %s", nDPIsrvd_enum_to_string(parse_ret)); + logger(1, "nDPIsrvd parse failed with: %s", nDPIsrvd_enum_to_string(parse_ret)); return 1; } } @@ -580,186 +909,570 @@ static int mainloop(int epollfd, struct nDPIsrvd_socket * const sock) return 0; } -static void collectd_map_token_to_stat(struct nDPIsrvd_socket * const sock, - struct nDPIsrvd_json_token const * const token, - struct json_stat_map const * const map, - size_t map_length) +static int collectd_map_to_stat(char const * const token_str, + size_t token_length, + struct global_map const * const map, + size_t map_length) { size_t i, null_i = map_length; - if (token == NULL) - { - return; - } - for (i = 0; i < map_length; ++i) { if (map[i].json_key == NULL) { null_i = i; - continue; + break; } - if (TOKEN_VALUE_EQUALS(sock, token, map[i].json_key, strlen(map[i].json_key)) != 0) + size_t key_length = strlen(map[i].json_key); + if (key_length == token_length && strncmp(map[i].json_key, token_str, token_length) == 0) { - (*map[i].collectd_stat)++; - return; + (*map[i].global_stat_inc)++; + return 0; } } - if (null_i < map_length) + if (null_i < map_length && map[null_i].global_stat_inc != NULL) { - (*map[null_i].collectd_stat)++; + (*map[null_i].global_stat_inc)++; + return 0; } + + return 1; } -static enum nDPIsrvd_callback_return collectd_json_callback(struct nDPIsrvd_socket * const sock, - struct nDPIsrvd_instance * const instance, - struct nDPIsrvd_thread_data * const thread_data, - struct nDPIsrvd_flow * const flow) +static int collectd_map_value_to_stat(struct nDPIsrvd_socket * const sock, + struct nDPIsrvd_json_token const * const token, + struct global_map const * const map, + size_t map_length) { - (void)instance; - (void)thread_data; + char const * value_str = NULL; + size_t value_length = 0; - struct nDPIsrvd_json_token const * const flow_event_name = TOKEN_GET_SZ(sock, "flow_event_name"); - struct flow_user_data * flow_user_data = NULL; + value_str = TOKEN_GET_VALUE(sock, token, &value_length); + if (value_length == 0 || value_str == NULL) + { + return 1; + } - collectd_statistics.json_lines++; - collectd_statistics.json_bytes += sock->buffer.json_message_length + NETWORK_BUFFER_LENGTH_DIGITS; + return collectd_map_to_stat(value_str, value_length, map, map_length); +} - struct nDPIsrvd_json_token const * const packet_event_name = TOKEN_GET_SZ(sock, "packet_event_name"); - if (packet_event_name != NULL) +static void collectd_unmap_flow_from_stat(struct flow_user_data * const flow_user_data) +{ + if (flow_user_data->is_ip4 != 0) { - collectd_map_token_to_stat(sock, packet_event_name, packet_event_map, nDPIsrvd_ARRAY_LENGTH(packet_event_map)); + COLLECTD_STATS_GAUGE_DEC(flow_l3_ip4_count); } - struct nDPIsrvd_json_token const * const daemon_event_name = TOKEN_GET_SZ(sock, "daemon_event_name"); - if (daemon_event_name != NULL) + if (flow_user_data->is_ip6 != 0) { - collectd_map_token_to_stat(sock, daemon_event_name, daemon_event_map, nDPIsrvd_ARRAY_LENGTH(daemon_event_map)); + COLLECTD_STATS_GAUGE_DEC(flow_l3_ip6_count); } - struct nDPIsrvd_json_token const * const error_event_id = TOKEN_GET_SZ(sock, "error_event_id"); - if (error_event_id != NULL) + if (flow_user_data->is_other_l3 != 0) { - nDPIsrvd_ull error_event_id_ull; - if (TOKEN_VALUE_TO_ULL(sock, error_event_id, &error_event_id_ull) != CONVERSION_OK) - { - return CALLBACK_ERROR; - } + COLLECTD_STATS_GAUGE_DEC(flow_l3_other_count); + } + + if (flow_user_data->is_tcp != 0) + { + COLLECTD_STATS_GAUGE_DEC(flow_l4_tcp_count); + } - collectd_statistics.error_count_sum++; - if (error_event_id_ull < ERROR_EVENT_ID_MAX) + if (flow_user_data->is_udp != 0) + { + COLLECTD_STATS_GAUGE_DEC(flow_l4_udp_count); + } + + if (flow_user_data->is_icmp != 0) + { + COLLECTD_STATS_GAUGE_DEC(flow_l4_icmp_count); + } + + if (flow_user_data->is_other_l4 != 0) + { + COLLECTD_STATS_GAUGE_DEC(flow_l4_other_count); + } + + if (flow_user_data->new_seen != 0) + { + COLLECTD_STATS_GAUGE_DEC(flow_active_count); + } + + if (flow_user_data->is_detected != 0) + { + COLLECTD_STATS_GAUGE_DEC(flow_detected_count); + } + + if (flow_user_data->is_guessed != 0) + { + COLLECTD_STATS_GAUGE_DEC(flow_guessed_count); + } + + if (flow_user_data->is_not_detected != 0) + { + COLLECTD_STATS_GAUGE_DEC(flow_not_detected_count); + } + + if (flow_user_data->is_info != 0) + { + COLLECTD_STATS_GAUGE_DEC(flow_state_info); + } + + if (flow_user_data->is_finished != 0) + { + COLLECTD_STATS_GAUGE_DEC(flow_state_finished); + } + + if (flow_user_data->breed > 0 && flow_user_data->breed_ndpid_invalid == 0 && + COLLECTD_STATS_MAP_NOTNULL(breeds_map, flow_user_data->breed) != 0) + { + COLLECTD_STATS_MAP_DEC(breeds_map, flow_user_data->breed); + } + + if (flow_user_data->category > 0 && flow_user_data->category_ndpid_invalid == 0 && + COLLECTD_STATS_MAP_NOTNULL(categories_map, flow_user_data->category) != 0) + { + COLLECTD_STATS_MAP_DEC(categories_map, flow_user_data->category); + } + + if (flow_user_data->confidence > 0 && flow_user_data->confidence_ndpid_invalid == 0 && + COLLECTD_STATS_MAP_NOTNULL(confidence_map, flow_user_data->confidence) != 0) + { + COLLECTD_STATS_MAP_DEC(confidence_map, flow_user_data->confidence); + } + + for (uint8_t i = 0; i < MAX_SEVERITIES_PER_FLOW; ++i) + { + if (flow_user_data->severities[i] > 0) { - collectd_statistics.error_count[error_event_id_ull]++; + COLLECTD_STATS_MAP_DEC(severity_map, flow_user_data->severities[i]); } - else + } + + for (uint8_t i = 0; i < MAX_RISKS_PER_FLOW; ++i) + { + if (flow_user_data->risks[i] > 0) { - collectd_statistics.error_unknown_count++; + COLLECTD_STATS_GAUGE_DEC(flow_risk_count[flow_user_data->risks[i]]); } } - if (flow != NULL) + if (flow_user_data->risk_ndpid_invalid != 0) { - flow_user_data = (struct flow_user_data *)flow->flow_user_data; + COLLECTD_STATS_GAUGE_DEC(flow_risk_unknown_count); } +} - if (flow_user_data != NULL) +static ssize_t collectd_map_index(char const * const json_key, + size_t key_length, + struct global_map const * const map, + size_t map_length) +{ + if (json_key == NULL || key_length == 0) { - nDPIsrvd_ull total_bytes_ull[2] = {0, 0}; + return -1; + } - if (TOKEN_VALUE_TO_ULL(sock, TOKEN_GET_SZ(sock, "flow_src_tot_l4_payload_len"), &total_bytes_ull[0]) == - CONVERSION_OK && - TOKEN_VALUE_TO_ULL(sock, TOKEN_GET_SZ(sock, "flow_dst_tot_l4_payload_len"), &total_bytes_ull[1]) == - CONVERSION_OK) + for (size_t i = 0; i < map_length; ++i) + { + if (map[i].json_key == NULL) { - collectd_statistics.flow_src_total_bytes += - total_bytes_ull[0] - flow_user_data->last_flow_src_l4_payload_len; - collectd_statistics.flow_dst_total_bytes += - total_bytes_ull[1] - flow_user_data->last_flow_dst_l4_payload_len; + continue; + } - flow_user_data->last_flow_src_l4_payload_len = total_bytes_ull[0]; - flow_user_data->last_flow_dst_l4_payload_len = total_bytes_ull[1]; + if (key_length == strlen(map[i].json_key) && strncmp(json_key, map[i].json_key, key_length) == 0) + { + return i; } } - collectd_map_token_to_stat(sock, flow_event_name, flow_event_map, nDPIsrvd_ARRAY_LENGTH(flow_event_map)); + return -1; +} + +static int collectd_map_flow_u8(struct nDPIsrvd_socket * const sock, + struct nDPIsrvd_json_token const * const token, + struct global_map const * const map, + size_t map_length, + uint8_t * const dest) +{ + if (token == NULL || dest == NULL) + { + return 1; + } + + size_t len; + char const * const str = TOKEN_GET_VALUE(sock, token, &len); + if (str == NULL || len == 0) + { + return 1; + } + + ssize_t const map_index = collectd_map_index(str, len, map, map_length); + if (map_index < 0 || map_index > UCHAR_MAX) + { + return 1; + } + + *dest = map_index + 1; + return 0; +} + +static void process_flow_stats(struct nDPIsrvd_socket * const sock, struct nDPIsrvd_flow * const flow) +{ + struct flow_user_data * flow_user_data; + struct nDPIsrvd_json_token const * const flow_event_name = TOKEN_GET_SZ(sock, "flow_event_name"); + struct nDPIsrvd_json_token const * const flow_state = TOKEN_GET_SZ(sock, "flow_state"); + nDPIsrvd_ull total_bytes_ull[2]; + + if (flow == NULL) + { + return; + } + flow_user_data = (struct flow_user_data *)flow->flow_user_data; + if (flow_user_data == NULL) + { + return; + } + if (TOKEN_VALUE_EQUALS_SZ(sock, flow_event_name, "new") != 0) { + flow_user_data->new_seen = 1; + COLLECTD_STATS_GAUGE_INC(flow_active_count); + struct nDPIsrvd_json_token const * const l3_proto = TOKEN_GET_SZ(sock, "l3_proto"); if (TOKEN_VALUE_EQUALS_SZ(sock, l3_proto, "ip4") != 0) { - collectd_statistics.flow_l3_ip4_count++; + flow_user_data->is_ip4 = 1; + COLLECTD_STATS_GAUGE_INC(flow_l3_ip4_count); } else if (TOKEN_VALUE_EQUALS_SZ(sock, l3_proto, "ip6") != 0) { - collectd_statistics.flow_l3_ip6_count++; + flow_user_data->is_ip6 = 1; + COLLECTD_STATS_GAUGE_INC(flow_l3_ip6_count); } else if (l3_proto != NULL) { - collectd_statistics.flow_l3_other_count++; + flow_user_data->is_other_l3 = 1; + COLLECTD_STATS_GAUGE_INC(flow_l3_other_count); } struct nDPIsrvd_json_token const * const l4_proto = TOKEN_GET_SZ(sock, "l4_proto"); if (TOKEN_VALUE_EQUALS_SZ(sock, l4_proto, "tcp") != 0) { - collectd_statistics.flow_l4_tcp_count++; + flow_user_data->is_tcp = 1; + COLLECTD_STATS_GAUGE_INC(flow_l4_tcp_count); } else if (TOKEN_VALUE_EQUALS_SZ(sock, l4_proto, "udp") != 0) { - collectd_statistics.flow_l4_udp_count++; + flow_user_data->is_udp = 1; + COLLECTD_STATS_GAUGE_INC(flow_l4_udp_count); } else if (TOKEN_VALUE_EQUALS_SZ(sock, l4_proto, "icmp") != 0) { - collectd_statistics.flow_l4_icmp_count++; + flow_user_data->is_icmp = 1; + COLLECTD_STATS_GAUGE_INC(flow_l4_icmp_count); } else if (l4_proto != NULL) { - collectd_statistics.flow_l4_other_count++; + flow_user_data->is_other_l4 = 1; + COLLECTD_STATS_GAUGE_INC(flow_l4_other_count); } } + else if (flow_user_data->new_seen == 0) + { + return; + } + + if (TOKEN_VALUE_EQUALS_SZ(sock, flow_event_name, "not-detected") != 0) + { + flow_user_data->is_not_detected = 1; + COLLECTD_STATS_GAUGE_INC(flow_not_detected_count); + } + else if (TOKEN_VALUE_EQUALS_SZ(sock, flow_event_name, "guessed") != 0) + { + flow_user_data->is_guessed = 1; + COLLECTD_STATS_GAUGE_INC(flow_guessed_count); + } else if (TOKEN_VALUE_EQUALS_SZ(sock, flow_event_name, "detected") != 0 || - TOKEN_VALUE_EQUALS_SZ(sock, flow_event_name, "detection-update") != 0 || - TOKEN_VALUE_EQUALS_SZ(sock, flow_event_name, "update") != 0) + TOKEN_VALUE_EQUALS_SZ(sock, flow_event_name, "detection-update") != 0) { struct nDPIsrvd_json_token const * const flow_risk = TOKEN_GET_SZ(sock, "ndpi", "flow_risk"); struct nDPIsrvd_json_token const * current = NULL; int next_child_index = -1; - if (flow_risk != NULL && flow_user_data != NULL) + if (flow_user_data->is_detected == 0) { - if (flow_user_data->detected_risks == 0) + flow_user_data->is_detected = 1; + COLLECTD_STATS_GAUGE_INC(flow_detected_count); + } + + if (flow_risk != NULL) + { + if (flow_user_data->risks[0] == 0) { - collectd_statistics.flow_risky_count++; + COLLECTD_STATS_COUNTER_INC(flow_risky_count); } while ((current = nDPIsrvd_get_next_token(sock, flow_risk, &next_child_index)) != NULL) { + size_t numeric_risk_len = 0; + char const * const numeric_risk_str = TOKEN_GET_KEY(sock, current, &numeric_risk_len); nDPIsrvd_ull numeric_risk_value = (nDPIsrvd_ull)-1; + char numeric_risk_buf[numeric_risk_len + 1]; - if (str_value_to_ull(TOKEN_GET_KEY(sock, current, NULL), &numeric_risk_value) == CONVERSION_OK) + if (numeric_risk_len > 0 && numeric_risk_str != NULL) { - if ((flow_user_data->detected_risks & (1ull << numeric_risk_value)) == 0) + strncpy(numeric_risk_buf, numeric_risk_str, numeric_risk_len); + numeric_risk_buf[numeric_risk_len] = '\0'; + + struct nDPIsrvd_json_token const * const severity = + TOKEN_GET_SZ(sock, "ndpi", "flow_risk", numeric_risk_buf, "severity"); + uint8_t severity_index; + + if (collectd_map_flow_u8( + sock, severity, severity_map, nDPIsrvd_ARRAY_LENGTH(severity_map), &severity_index) != 0) + { + severity_index = 0; + } + + if (severity_index != 0) + { + for (uint8_t i = 0; i < MAX_SEVERITIES_PER_FLOW; ++i) + { + if (flow_user_data->severities[i] != 0) + { + continue; + } + if (flow_user_data->severities[i] == severity_index) + { + break; + } + + if (collectd_map_value_to_stat( + sock, severity, severity_map, nDPIsrvd_ARRAY_LENGTH(severity_map)) != 0) + { + severity_index = 0; + break; + } + flow_user_data->severities[i] = severity_index; + break; + } + } + if (severity_index == 0) + { + size_t value_len = 0; + char const * const value_str = TOKEN_GET_VALUE(sock, severity, &value_len); + + if (value_len > 0 && value_str != NULL) + { + logger(1, + "Unknown/Invalid JSON value for key 'ndpi','breed': %.*s", + (int)value_len, + value_str); + } + } + + if (str_value_to_ull(numeric_risk_str, &numeric_risk_value) == CONVERSION_OK) { if (numeric_risk_value < NDPI_MAX_RISK && numeric_risk_value > 0) { - collectd_statistics.flow_risk_count[numeric_risk_value - 1]++; - flow_user_data->detected_risks |= (1ull << (numeric_risk_value - 1)); + for (uint8_t i = 0; i < MAX_RISKS_PER_FLOW; ++i) + { + if (flow_user_data->risks[i] != 0) + { + continue; + } + if (flow_user_data->risks[i] == numeric_risk_value) + { + break; + } + + COLLECTD_STATS_GAUGE_INC(flow_risk_count[numeric_risk_value]); + flow_user_data->risks[i] = numeric_risk_value; + break; + } } - else + else if (flow_user_data->risk_ndpid_invalid == 0) { - collectd_statistics.flow_risk_unknown_count++; + flow_user_data->risk_ndpid_invalid = 1; + COLLECTD_STATS_GAUGE_INC(flow_risk_unknown_count); } } + else + { + logger(1, "Invalid numeric risk value: %s", numeric_risk_buf); + } + } + else + { + logger(1, "%s", "Missing numeric risk value"); + } + } + } + + if (flow_user_data->breed == 0 && flow_user_data->breed_ndpid_invalid == 0) + { + struct nDPIsrvd_json_token const * const breed = TOKEN_GET_SZ(sock, "ndpi", "breed"); + if (collectd_map_flow_u8( + sock, breed, breeds_map, nDPIsrvd_ARRAY_LENGTH(breeds_map), &flow_user_data->breed) != 0 || + collectd_map_value_to_stat(sock, breed, breeds_map, nDPIsrvd_ARRAY_LENGTH(breeds_map)) != 0) + { + size_t value_len = 0; + char const * const value_str = TOKEN_GET_VALUE(sock, breed, &value_len); + + flow_user_data->breed = 0; + flow_user_data->breed_ndpid_invalid = 1; + if (value_len > 0 && value_str != NULL) + { + logger(1, "Unknown/Invalid JSON value for key 'ndpi','breed': %.*s", (int)value_len, value_str); + } + } + } + + if (flow_user_data->category == 0 && flow_user_data->category_ndpid_invalid == 0) + { + struct nDPIsrvd_json_token const * const category = TOKEN_GET_SZ(sock, "ndpi", "category"); + if (collectd_map_flow_u8( + sock, category, categories_map, nDPIsrvd_ARRAY_LENGTH(categories_map), &flow_user_data->category) != + 0 || + collectd_map_value_to_stat(sock, category, categories_map, nDPIsrvd_ARRAY_LENGTH(categories_map)) != 0) + { + size_t value_len = 0; + char const * const value_str = TOKEN_GET_VALUE(sock, category, &value_len); + + flow_user_data->category = 0; + flow_user_data->category_ndpid_invalid = 1; + if (value_len > 0 && value_str != NULL) + { + logger(1, "Unknown/Invalid JSON value for key 'ndpi','category': %.*s", (int)value_len, value_str); } } } - struct nDPIsrvd_json_token const * const breed = TOKEN_GET_SZ(sock, "ndpi", "breed"); - collectd_map_token_to_stat(sock, breed, breeds_map, nDPIsrvd_ARRAY_LENGTH(breeds_map)); + if (flow_user_data->confidence == 0 && flow_user_data->confidence_ndpid_invalid == 0) + { + struct nDPIsrvd_json_token const * const token = TOKEN_GET_SZ(sock, "ndpi", "confidence"); + struct nDPIsrvd_json_token const * current = NULL; + int next_child_index = -1; + + if ((current = nDPIsrvd_get_next_token(sock, token, &next_child_index)) == NULL) + { + flow_user_data->confidence_ndpid_invalid = 1; + } + else if (nDPIsrvd_get_next_token(sock, token, &next_child_index) == NULL) + { + if (collectd_map_flow_u8(sock, + current, + confidence_map, + nDPIsrvd_ARRAY_LENGTH(confidence_map), + &flow_user_data->confidence) != 0 || + collectd_map_value_to_stat(sock, current, confidence_map, nDPIsrvd_ARRAY_LENGTH(confidence_map)) != + 0) + { + flow_user_data->confidence = 0; + flow_user_data->confidence_ndpid_invalid = 1; + } + } + else + { + flow_user_data->confidence_ndpid_invalid = 1; + } + + if (flow_user_data->confidence_ndpid_invalid != 0) + { + size_t value_len = 0; + char const * const value_str = TOKEN_GET_VALUE(sock, current, &value_len); + + logger(1, "Unknown/Invalid JSON value for key 'ndpi','confidence': %.*s", (int)value_len, value_str); + } + } + } + + if (TOKEN_VALUE_EQUALS_SZ(sock, flow_state, "info") != 0) + { + if (flow_user_data->is_info == 0) + { + flow_user_data->is_info = 1; + COLLECTD_STATS_GAUGE_INC(flow_state_info); + } + } + else if (TOKEN_VALUE_EQUALS_SZ(sock, flow_state, "finished") != 0) + { + if (flow_user_data->is_finished == 0) + { + if (flow_user_data->is_info != 0) + { + flow_user_data->is_info = 0; + COLLECTD_STATS_GAUGE_RES(flow_state_info); + } + flow_user_data->is_finished = 1; + COLLECTD_STATS_GAUGE_INC(flow_state_finished); + } + } + + if (TOKEN_VALUE_TO_ULL(sock, TOKEN_GET_SZ(sock, "flow_src_tot_l4_payload_len"), &total_bytes_ull[0]) == + CONVERSION_OK && + TOKEN_VALUE_TO_ULL(sock, TOKEN_GET_SZ(sock, "flow_dst_tot_l4_payload_len"), &total_bytes_ull[1]) == + CONVERSION_OK) + { + collectd_statistics.counters.flow_src_total_bytes += + total_bytes_ull[0] - flow_user_data->last_flow_src_l4_payload_len; + collectd_statistics.counters.flow_dst_total_bytes += + total_bytes_ull[1] - flow_user_data->last_flow_dst_l4_payload_len; + + flow_user_data->last_flow_src_l4_payload_len = total_bytes_ull[0]; + flow_user_data->last_flow_dst_l4_payload_len = total_bytes_ull[1]; + } + + if (TOKEN_VALUE_EQUALS_SZ(sock, flow_event_name, "end") != 0 || + TOKEN_VALUE_EQUALS_SZ(sock, flow_event_name, "idle") != 0) + { + collectd_unmap_flow_from_stat(flow_user_data); + } +} + +static enum nDPIsrvd_callback_return collectd_json_callback(struct nDPIsrvd_socket * const sock, + struct nDPIsrvd_instance * const instance, + struct nDPIsrvd_thread_data * const thread_data, + struct nDPIsrvd_flow * const flow) +{ + (void)instance; + (void)thread_data; + + struct nDPIsrvd_json_token const * const flow_event = TOKEN_GET_SZ(sock, "flow_event_name"); + struct nDPIsrvd_json_token const * const packet_event = TOKEN_GET_SZ(sock, "packet_event_name"); + struct nDPIsrvd_json_token const * const daemon_event = TOKEN_GET_SZ(sock, "daemon_event_name"); + struct nDPIsrvd_json_token const * const error_event = TOKEN_GET_SZ(sock, "error_event_name"); + + COLLECTD_STATS_COUNTER_INC(json_lines); + collectd_statistics.counters.json_bytes += sock->buffer.json_message_length + NETWORK_BUFFER_LENGTH_DIGITS; + + process_flow_stats(sock, flow); + + if (flow_event != NULL && + collectd_map_value_to_stat(sock, flow_event, flow_event_map, nDPIsrvd_ARRAY_LENGTH(flow_event_map)) != 0) + { + logger(1, "%s", "Unknown flow_event_name"); + } - struct nDPIsrvd_json_token const * const category = TOKEN_GET_SZ(sock, "ndpi", "category"); - collectd_map_token_to_stat(sock, category, categories_map, nDPIsrvd_ARRAY_LENGTH(categories_map)); + if (packet_event != NULL && + collectd_map_value_to_stat(sock, packet_event, packet_event_map, nDPIsrvd_ARRAY_LENGTH(packet_event_map)) != 0) + { + logger(1, "%s", "Unknown packet_event_name"); + } + + if (daemon_event != NULL && + collectd_map_value_to_stat(sock, daemon_event, daemon_event_map, nDPIsrvd_ARRAY_LENGTH(daemon_event_map)) != 0) + { + logger(1, "%s", "Unknown daemon_event_name"); + } + + if (error_event != NULL && + collectd_map_value_to_stat(sock, error_event, error_event_map, nDPIsrvd_ARRAY_LENGTH(error_event_map)) != 0) + { + logger(1, "%s", "Unknown error_event_name"); } return CALLBACK_OK; @@ -770,13 +1483,13 @@ int main(int argc, char ** argv) enum nDPIsrvd_connect_return connect_ret; int retval = 1, epollfd = -1; - openlog("nDPIsrvd-collectd", LOG_CONS, LOG_DAEMON); + init_logging("nDPIsrvd-collectd"); struct nDPIsrvd_socket * sock = nDPIsrvd_socket_init(0, 0, 0, sizeof(struct flow_user_data), collectd_json_callback, NULL, NULL); if (sock == NULL) { - LOG(LOG_DAEMON | LOG_ERR, "%s", "nDPIsrvd socket memory allocation failed!"); + LOGGER_EARLY(1, "%s", "nDPIsrvd socket memory allocation failed!"); return 1; } @@ -787,33 +1500,32 @@ int main(int argc, char ** argv) if (getenv("COLLECTD_HOSTNAME") == NULL && getenv("COLLECTD_INTERVAL") == NULL) { - LOG(LOG_DAEMON | LOG_NOTICE, "Recv buffer size: %u", NETWORK_BUFFER_MAX_SIZE); - LOG(LOG_DAEMON | LOG_NOTICE, "Connecting to `%s'..", serv_optarg); + LOGGER_EARLY(0, "Recv buffer size: %u", NETWORK_BUFFER_MAX_SIZE); + LOGGER_EARLY(0, "Connecting to `%s'..", serv_optarg); } else { - quiet = 1; - LOG(LOG_DAEMON | LOG_NOTICE, "Collectd hostname: %s", getenv("COLLECTD_HOSTNAME")); - LOG(LOG_DAEMON | LOG_NOTICE, "Collectd interval: %llu", collectd_interval_ull); + LOGGER_EARLY(0, "Collectd hostname: %s", getenv("COLLECTD_HOSTNAME")); + LOGGER_EARLY(0, "Collectd interval: %llu", collectd_interval_ull); } if (setvbuf(stdout, NULL, _IONBF, 0) != 0) { - LOG(LOG_DAEMON | LOG_ERR, - "Could not set stdout unbuffered: %s. Collectd may receive too old PUTVALs and complain.", - strerror(errno)); + LOGGER_EARLY(1, + "Could not set stdout unbuffered: %s. Collectd may receive too old PUTVALs and complain.", + strerror(errno)); } connect_ret = nDPIsrvd_connect(sock); if (connect_ret != CONNECT_OK) { - LOG(LOG_DAEMON | LOG_ERR, "nDPIsrvd socket connect to %s failed!", serv_optarg); + LOGGER_EARLY(1, "nDPIsrvd socket connect to %s failed!", serv_optarg); goto failure; } if (nDPIsrvd_set_nonblock(sock) != 0) { - LOG(LOG_DAEMON | LOG_ERR, "nDPIsrvd set nonblock failed: %s", strerror(errno)); + LOGGER_EARLY(1, "nDPIsrvd set nonblock failed: %s", strerror(errno)); goto failure; } @@ -826,13 +1538,13 @@ int main(int argc, char ** argv) epollfd = epoll_create1(0); if (epollfd < 0) { - LOG(LOG_DAEMON | LOG_ERR, "Error creating epoll: %s", strerror(errno)); + LOGGER_EARLY(1, "Error creating epoll: %s", strerror(errno)); goto failure; } if (create_collectd_timer() != 0) { - LOG(LOG_DAEMON | LOG_ERR, "Error creating timer: %s", strerror(errno)); + LOGGER_EARLY(1, "Error creating timer: %s", strerror(errno)); goto failure; } @@ -840,7 +1552,7 @@ int main(int argc, char ** argv) struct epoll_event timer_event = {.data.fd = collectd_timerfd, .events = EPOLLIN}; if (epoll_ctl(epollfd, EPOLL_CTL_ADD, collectd_timerfd, &timer_event) < 0) { - LOG(LOG_DAEMON | LOG_ERR, "Error adding JSON fd to epoll: %s", strerror(errno)); + LOGGER_EARLY(1, "Error adding JSON fd to epoll: %s", strerror(errno)); goto failure; } } @@ -849,12 +1561,12 @@ int main(int argc, char ** argv) struct epoll_event socket_event = {.data.fd = sock->fd, .events = EPOLLIN}; if (epoll_ctl(epollfd, EPOLL_CTL_ADD, sock->fd, &socket_event) < 0) { - LOG(LOG_DAEMON | LOG_ERR, "Error adding nDPIsrvd socket fd to epoll: %s", strerror(errno)); + LOGGER_EARLY(1, "Error adding nDPIsrvd socket fd to epoll: %s", strerror(errno)); goto failure; } } - LOG(LOG_DAEMON | LOG_NOTICE, "%s", "Initialization succeeded."); + logger(0, "%s", "Initialization succeeded."); retval = mainloop(epollfd, sock); if (getenv("COLLECTD_INTERVAL") == NULL) @@ -866,7 +1578,7 @@ failure: nDPIsrvd_socket_free(&sock); close(collectd_timerfd); close(epollfd); - closelog(); + shutdown_logging(); return retval; } |