diff options
author | Yağmur Oymak <yagmur.oymak@gmail.com> | 2019-07-24 19:16:44 +0300 |
---|---|---|
committer | Yağmur Oymak <yagmur.oymak@gmail.com> | 2019-07-24 19:16:44 +0300 |
commit | 786ea3c0f9d862546c56bed15fce574eec3991ee (patch) | |
tree | 9eb7a590d309bf433ac41caf18c9f2b0103bad22 /src | |
parent | 9a899c54c927bb6012ed39e42c9be9cd9c4c7151 (diff) | |
parent | 21485683475ab6b3bb4468f142843d5f15f412f8 (diff) |
Merge branch 'dev' into wireguard
Conflicts:
src/include/ndpi_protocols.h
src/lib/ndpi_main.c
Get upstream developments.
Diffstat (limited to 'src')
-rw-r--r-- | src/include/ndpi_api.h | 8 | ||||
-rw-r--r-- | src/include/ndpi_classify.h | 92 | ||||
-rw-r--r-- | src/include/ndpi_protocol_ids.h | 1 | ||||
-rw-r--r-- | src/include/ndpi_protocols.h | 5 | ||||
-rw-r--r-- | src/include/ndpi_typedefs.h | 45 | ||||
-rw-r--r-- | src/lib/Makefile.in | 2 | ||||
-rw-r--r-- | src/lib/ndpi_classify.c | 673 | ||||
-rw-r--r-- | src/lib/ndpi_content_match.c.inc | 28 | ||||
-rw-r--r-- | src/lib/ndpi_main.c | 28 | ||||
-rw-r--r-- | src/lib/ndpi_utils.c | 138 | ||||
-rw-r--r-- | src/lib/protocols/amazon_video.c | 81 | ||||
-rw-r--r-- | src/lib/protocols/stun.c | 48 | ||||
-rw-r--r-- | src/lib/protocols/targus_getdata.c | 77 | ||||
-rw-r--r-- | src/lib/protocols/xbox.c | 7 |
14 files changed, 1222 insertions, 11 deletions
diff --git a/src/include/ndpi_api.h b/src/include/ndpi_api.h index 41ae2fbaa..20dd048ad 100644 --- a/src/include/ndpi_api.h +++ b/src/include/ndpi_api.h @@ -821,6 +821,10 @@ extern "C" { int ndpi_init_serializer(ndpi_serializer *serializer, ndpi_serialization_format fmt); void ndpi_term_serializer(ndpi_serializer *serializer); void ndpi_reset_serializer(ndpi_serializer *serializer); + int ndpi_serialize_string_int32(ndpi_serializer *serializer, + const char *key, int32_t value); + int ndpi_serialize_string_int64(ndpi_serializer *serializer, + const char *key, int64_t value); int ndpi_serialize_uint32_uint32(ndpi_serializer *serializer, u_int32_t key, u_int32_t value); int ndpi_serialize_uint32_uint64(ndpi_serializer *serializer, @@ -850,6 +854,10 @@ extern "C" { u_int32_t *key, u_int64_t *value); int ndpi_deserialize_uint32_string(ndpi_deserializer *deserializer, u_int32_t *key, ndpi_string *value); + int ndpi_deserialize_string_int32(ndpi_deserializer *deserializer, + ndpi_string *key, int32_t *value); + int ndpi_deserialize_string_int64(ndpi_deserializer *deserializer, + ndpi_string *key, int64_t *value); int ndpi_deserialize_string_uint32(ndpi_deserializer *deserializer, ndpi_string *key, u_int32_t *value); int ndpi_deserialize_string_uint64(ndpi_deserializer *deserializer, diff --git a/src/include/ndpi_classify.h b/src/include/ndpi_classify.h new file mode 100644 index 000000000..9b5f2841f --- /dev/null +++ b/src/include/ndpi_classify.h @@ -0,0 +1,92 @@ +/* + * + * Copyright (c) 2016 Cisco Systems, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * + * Neither the name of the Cisco Systems, Inc. nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, + * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + * OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +/** + * \file ndpi_classify.h + * + * \brief header file for inline Classification functionality + */ + +#ifndef NDPI_CLASSIFY_H +#define NDPI_CLASSIFY_H + +#include <stdint.h> +#include <sys/time.h> + +/* constants */ +#define NUM_PARAMETERS_SPLT_LOGREG 208 +#define NUM_PARAMETERS_BD_LOGREG 464 +#define MC_BINS_LEN 10 +#define MC_BINS_TIME 10 +#define MC_BIN_SIZE_TIME 50 +#define MC_BIN_SIZE_LEN 150 +#define MAX_BIN_LEN 1500 +#define NUM_BD_VALUES 256 +#define NDPI_TIMESTAMP_LEN 64 + +/** Classifier parameter type codes */ +typedef enum { + SPLT_PARAM_TYPE = 0, + BD_PARAM_TYPE = 1 +} classifier_type_codes_t; + +extern float parameters_bd[NUM_PARAMETERS_BD_LOGREG]; +extern float parameters_splt[NUM_PARAMETERS_SPLT_LOGREG]; + +/* Classifier functions */ +float ndpi_classify(const unsigned short *pkt_len, const struct timeval *pkt_time, + const unsigned short *pkt_len_twin, const struct timeval *pkt_time_twin, + struct timeval start_time, struct timeval start_time_twin, uint32_t max_num_pkt_len, + uint16_t sp, uint16_t dp, uint32_t op, uint32_t ip, uint32_t np_o, uint32_t np_i, + uint32_t ob, uint32_t ib, uint16_t use_bd, const uint32_t *bd, const uint32_t *bd_t); + +void ndpi_merge_splt_arrays(const uint16_t *pkt_len, const struct timeval *pkt_time, + const uint16_t *pkt_len_twin, const struct timeval *pkt_time_twin, + struct timeval start_time, struct timeval start_time_twin, + uint16_t s_idx, uint16_t r_idx, + uint16_t *merged_lens, uint16_t *merged_times); + +void ndpi_update_params(classifier_type_codes_t param_type, const char *param_file); + +void ndpi_flow_info_freer(void *node); +unsigned int ndpi_timer_eq(const struct timeval *a, const struct timeval *b); +unsigned int ndpi_timer_lt(const struct timeval *a, const struct timeval *b); +void ndpi_timer_sub(const struct timeval *a, const struct timeval *b, struct timeval *result); +void ndpi_timer_clear(struct timeval *a); +unsigned int ndpi_timeval_to_milliseconds(struct timeval ts); +void ndpi_log_timestamp(char *log_ts, u_int log_ts_len); + +#endif /* NDPI_CLASSIFY_H */ diff --git a/src/include/ndpi_protocol_ids.h b/src/include/ndpi_protocol_ids.h index 4ceb61112..4ad618e3e 100644 --- a/src/include/ndpi_protocol_ids.h +++ b/src/include/ndpi_protocol_ids.h @@ -277,6 +277,7 @@ typedef enum { NDPI_PROTOCOL_AMAZON_VIDEO = 240, NDPI_PROTOCOL_GOOGLE_DOCS = 241, NDPI_PROTOCOL_WHATSAPP_FILES = 242, /* Videos, pictures, voice messages... */ + NDPI_PROTOCOL_TARGUS_GETDATA = 243, /* IMPORTANT before allocating a new identifier please fill up diff --git a/src/include/ndpi_protocols.h b/src/include/ndpi_protocols.h index 7991660a9..187fbec13 100644 --- a/src/include/ndpi_protocols.h +++ b/src/include/ndpi_protocols.h @@ -206,6 +206,9 @@ void ndpi_search_ajp(struct ndpi_detection_module_struct *ndpi_struct, struct nd void ndpi_search_memcached(struct ndpi_detection_module_struct *ndpi_struct, struct ndpi_flow_struct *flow); void ndpi_search_nest_log_sink(struct ndpi_detection_module_struct *ndpi_struct, struct ndpi_flow_struct *flow); void ndpi_search_wireguard(struct ndpi_detection_module_struct *ndpi_struct, struct ndpi_flow_struct *flow); +void ndpi_search_targus_getdata(struct ndpi_detection_module_struct *ndpi_struct, struct ndpi_flow_struct *flow); +void ndpi_search_apple_push(struct ndpi_detection_module_struct *ndpi_struct, struct ndpi_flow_struct *flow); +void ndpi_search_amazon_video(struct ndpi_detection_module_struct *ndpi_struct, struct ndpi_flow_struct *flow); /* --- INIT FUNCTIONS --- */ void init_diameter_dissector(struct ndpi_detection_module_struct *ndpi_struct, u_int32_t *id, NDPI_PROTOCOL_BITMASK *detection_bitmask); void init_afp_dissector(struct ndpi_detection_module_struct *ndpi_struct, u_int32_t *id, NDPI_PROTOCOL_BITMASK *detection_bitmask); @@ -358,6 +361,7 @@ void init_nintendo_dissector(struct ndpi_detection_module_struct *ndpi_struct, u void init_csgo_dissector(struct ndpi_detection_module_struct *ndpi_struct, u_int32_t *id, NDPI_PROTOCOL_BITMASK *detection_bitmask); void init_checkmk_dissector(struct ndpi_detection_module_struct *ndpi_struct, u_int32_t *id, NDPI_PROTOCOL_BITMASK *detection_bitmask); void init_apple_push_dissector(struct ndpi_detection_module_struct *ndpi_struct, u_int32_t *id, NDPI_PROTOCOL_BITMASK *detection_bitmask); +void init_amazon_video_dissector(struct ndpi_detection_module_struct *ndpi_struct, u_int32_t *id, NDPI_PROTOCOL_BITMASK *detection_bitmask); void init_whatsapp_dissector(struct ndpi_detection_module_struct *ndpi_struct, u_int32_t *id, NDPI_PROTOCOL_BITMASK *detection_bitmask); void init_ajp_dissector(struct ndpi_detection_module_struct *ndpi_struct, u_int32_t *id, NDPI_PROTOCOL_BITMASK *detection_bitmask); void init_fbzero_dissector(struct ndpi_detection_module_struct *ndpi_struct, u_int32_t *id, NDPI_PROTOCOL_BITMASK *detection_bitmask); @@ -367,5 +371,6 @@ void init_ookla_dissector(struct ndpi_detection_module_struct *ndpi_struct, u_in void init_modbus_dissector(struct ndpi_detection_module_struct *ndpi_struct, u_int32_t *id, NDPI_PROTOCOL_BITMASK *detection_bitmask); void init_line_dissector(struct ndpi_detection_module_struct *ndpi_struct, u_int32_t *id, NDPI_PROTOCOL_BITMASK *detection_bitmask); void init_wireguard_dissector(struct ndpi_detection_module_struct *ndpi_struct, u_int32_t *id, NDPI_PROTOCOL_BITMASK *detection_bitmask); +void init_targus_getdata_dissector(struct ndpi_detection_module_struct *ndpi_struct, u_int32_t *id, NDPI_PROTOCOL_BITMASK *detection_bitmask); #endif /* __NDPI_PROTOCOLS_H__ */ diff --git a/src/include/ndpi_typedefs.h b/src/include/ndpi_typedefs.h index f576834cc..e240f7847 100644 --- a/src/include/ndpi_typedefs.h +++ b/src/include/ndpi_typedefs.h @@ -122,6 +122,49 @@ struct ndpi_ethhdr u_int16_t h_proto; /* data length (<= 1500) or type ID proto (>=1536) */ } PACK_OFF; +/* +++++++++++++++ ARP header +++++++++++++++ */ +PACK_ON +struct ndpi_arphdr { + u_int16_t ar_hrd;/* Format of hardware address. */ + u_int16_t ar_pro;/* Format of protocol address. */ + u_int8_t ar_hln;/* Length of hardware address. */ + u_int8_t ar_pln;/* Length of protocol address. */ + u_int16_t ar_op;/* ARP opcode (command). */ + u_char arp_sha[6];/* sender hardware address */ + u_int32_t arp_spa;/* sender protocol address */ + u_char arp_tha[6];/* target hardware address */ + u_int32_t arp_tpa;/* target protocol address */ +} PACK_OFF; + +/* +++++++++++++++ DHCP header +++++++++++++++ */ +PACK_ON +struct ndpi_dhcphdr { + u_int8_t msgType; + u_int8_t htype; + u_int8_t hlen; + u_int8_t hops; + u_int32_t xid;/* 4 */ + u_int16_t secs;/* 8 */ + u_int16_t flags; + u_int32_t ciaddr;/* 12 */ + u_int32_t yiaddr;/* 16 */ + u_int32_t siaddr;/* 20 */ + u_int32_t giaddr;/* 24 */ + u_int8_t chaddr[16]; /* 28 */ + u_int8_t sname[64]; /* 44 */ + u_int8_t file[128]; /* 108 */ + u_int32_t magic; /* 236 */ + u_int8_t options[308]; +} PACK_OFF; + +/* +++++++++++++++ MDNS rsp header +++++++++++++++ */ +PACK_ON +struct ndpi_mdns_rsp_entry { + u_int16_t rsp_type, rsp_class; + u_int32_t ttl; + u_int16_t data_len; +} PACK_OFF; + /* +++++++++++++++++++ LLC header (IEEE 802.2) ++++++++++++++++ */ PACK_ON @@ -1275,6 +1318,8 @@ typedef enum { ndpi_serialization_uint32_uint32, ndpi_serialization_uint32_uint64, ndpi_serialization_uint32_string, + ndpi_serialization_string_int32, + ndpi_serialization_string_int64, ndpi_serialization_string_uint32, ndpi_serialization_string_uint64, ndpi_serialization_string_string, diff --git a/src/lib/Makefile.in b/src/lib/Makefile.in index 3a386fd5d..1a884ac9d 100644 --- a/src/lib/Makefile.in +++ b/src/lib/Makefile.in @@ -17,7 +17,7 @@ CC = @CC@ CFLAGS += -fPIC -DPIC -I../include -Ithird_party/include -DNDPI_LIB_COMPILATION -O2 -g -Wall RANLIB = ranlib -OBJECTS = $(patsubst protocols/%.c, protocols/%.o, $(wildcard protocols/*.c)) $(patsubst third_party/src/%.c, third_party/src/%.o, $(wildcard third_party/src/*.c)) ndpi_main.o ndpi_utils.o +OBJECTS = $(patsubst protocols/%.c, protocols/%.o, $(wildcard protocols/*.c)) $(patsubst third_party/src/%.c, third_party/src/%.o, $(wildcard third_party/src/*.c)) $(patsubst ./%.c, ./%.o, $(wildcard ./*.c)) HEADERS = $(wildcard ../include/*.h) NDPI_VERSION_MAJOR = @NDPI_MAJOR@ NDPI_LIB_STATIC = libndpi.a diff --git a/src/lib/ndpi_classify.c b/src/lib/ndpi_classify.c new file mode 100644 index 000000000..79cf1c234 --- /dev/null +++ b/src/lib/ndpi_classify.c @@ -0,0 +1,673 @@ +/* + * + * Copyright (c) 2016 Cisco Systems, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * + * Neither the name of the Cisco Systems, Inc. nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, + * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + * OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +/** + * \file ndpi_classify.c + * + * \brief contains the functionality for inline classification + * + */ + +#define _GNU_SOURCE +#ifdef HAVE_CONFIG_H +#include "ndpi_config.h" +#endif + +#include <stdio.h> +#include <ctype.h> +#include <sys/time.h> +#include <stdlib.h> +#include <stdint.h> +#include <math.h> + +#include "ndpi_main.h" +#include "ndpi_classify.h" + +/** finds the minimum value between to inputs */ +#define min(a,b) \ + ({ __typeof__ (a) _a = (a); \ + __typeof__ (b) _b = (b); \ + _a < _b ? _a : _b; }) + +//bias (1) + w (207) +//const float ndpi_parameters_splt[NUM_PARAMETERS_SPLT_LOGREG] = { +float ndpi_parameters_splt[NUM_PARAMETERS_SPLT_LOGREG] = { + 1.870162393265777379e+00, -4.795306993214020408e-05, -1.734180056229888626e-04, -6.750871045910851378e-04, + 5.175991233904169049e-04, 3.526042198693187802e-07, -2.903366739676974950e-07, -1.415422572109461820e-06, + -1.771571627605233568e+00, 1.620550564201104216e+00, -4.612754771764762118e-01, 3.239944708329216994e+00, + 2.798317033823678024e+00, 0.000000000000000000e+00, 6.076539623210191365e+00, 3.308503132975965322e+00, + -1.092831892216604983e-01, 2.982742154817296765e+00, 1.660969487778582554e+00, -3.456805843507989584e-01, + 1.348166013591903800e+00, 2.574204101170268211e-01, 2.610145601469008980e+00, 1.020576625389262970e+00, + 3.671704233284687646e+00, 6.443945529005814521e-01, 2.252762512697924647e-03, 2.204985803678578549e+00, + -2.175241664145967091e-01, -1.141779409733734239e-03, -4.170326358555080049e+00, -6.042082896847342788e-01, + 6.081676509559893473e-01, 1.067078220396491028e+00, 3.836704027854674903e-01, 0.000000000000000000e+00, + 2.146120236132928460e-02, 2.432303290345616098e+00, -2.103340262991825860e+00, -1.744126902195192397e+00, + -2.623163496699073338e+00, -3.407875120566610239e+00, -7.964525551010913640e-01, -1.404640840577571437e+00, + 0.000000000000000000e+00, 0.000000000000000000e+00, 1.880185666666627593e-04, 1.282810736369378146e+00, + -9.641782614904219617e-01, 4.403448314292253141e-01, -3.657774135738374455e-02, 0.000000000000000000e+00, + -5.459785394047789175e-02, 0.000000000000000000e+00, -1.282088906132429429e+00, 4.860766361538500224e-02, + -2.169499256433678802e+00, 4.022086644863120397e-01, -4.914517759289173116e-01, -7.589910385869531595e-01, + 0.000000000000000000e+00, 0.000000000000000000e+00, 0.000000000000000000e+00, -8.666757574333044944e-01, + 0.000000000000000000e+00, 0.000000000000000000e+00, 0.000000000000000000e+00, 1.311256230786839394e+00, + -1.914518488914755068e+00, 1.511887239039312325e+00, 2.178209608648221440e+00, -7.023826963231178944e-01, + 0.000000000000000000e+00, 2.007745367392351800e-03, -1.741067164349403007e-01, -8.226617550716192051e-02, + -3.117883014105701456e+00, 6.628843265383883576e-01, -2.160748016600245514e+00, -1.405593258704527670e+00, + 1.183206385957070061e+00, 2.757013165261792964e-01, 0.000000000000000000e+00, 0.000000000000000000e+00, + 0.000000000000000000e+00, -1.811005099043709210e+00, -8.455596144009722703e-01, 1.001251761183534761e+00, + -3.939531953397222841e-01, 1.993035329450950854e+00, -1.019523996210594863e+00, -1.755881638655008015e-01, + -1.787388667240087603e+00, -2.430688550462867248e-01, 0.000000000000000000e+00, -1.884287953600421561e+00, + -2.751870090435341254e+00, 1.697458788143338504e+00, 1.849679594159958553e-03, 4.943355128829073908e-01, + 7.369008876756165671e-01, 4.693987613154877003e+00, 3.064798194276571741e-01, 5.387093625046185386e-01, + 0.000000000000000000e+00, 0.000000000000000000e+00, 3.624061495761899732e+00, 3.074260971046838264e-01, + -1.431368135826769805e+00, 6.994209180148978078e-01, 1.790376540283850959e+00, 1.524331645466284968e+00, + 1.358206060175735086e+00, 1.467425790557983944e+00, 1.186415624035605187e+00, 5.323820984869050976e-01, + 8.591955162076543237e-03, 3.118816279950378800e-01, -4.062663013982938942e-01, 2.242707735666635838e+00, + 7.686176932110666549e-01, -1.560810210584786528e-01, 3.540092084282713825e-01, 0.000000000000000000e+00, + 3.097791434293225565e-01, 0.000000000000000000e+00, 0.000000000000000000e+00, -6.986278186857963757e-01, + -7.517062056086308564e-02, 7.074462217191725966e-01, 2.652408529563320627e+00, 2.147183236174156074e-01, + 0.000000000000000000e+00, -5.440998469665543347e-01, -4.689480062559393640e-03, 0.000000000000000000e+00, + 0.000000000000000000e+00, -1.086404167506188401e+00, 2.630806090789038487e-01, -7.025803998688389118e-01, + 0.000000000000000000e+00, 3.643784401628049618e+00, 0.000000000000000000e+00, 0.000000000000000000e+00, + -2.302902226543305508e-01, 0.000000000000000000e+00, 0.000000000000000000e+00, -1.189427790184047318e+00, + -1.628347806351332916e-01, 1.811447446865056266e-01, -1.013719736818966766e+00, 0.000000000000000000e+00, + 2.160057499014520488e+00, 5.515077573228228669e-01, 2.124699422003203608e+00, 0.000000000000000000e+00, + 0.000000000000000000e+00, -7.125921670169992339e-01, 1.539140748884923991e-02, 8.977156480543774242e-02, + -1.449316192295113881e+00, -5.367207427320878910e-03, 7.097016086830079118e-01, 1.787774315099208255e+00, + 1.005405396661134043e+00, 1.444517882048796054e+00, 0.000000000000000000e+00, -1.009699404577948600e+00, + -1.079157463961748942e+00, -2.360021607239124741e-01, -1.267330554215283733e+00, 0.000000000000000000e+00, + 0.000000000000000000e+00, 0.000000000000000000e+00, 6.230706551651569169e-01, 0.000000000000000000e+00, + 0.000000000000000000e+00, -2.698683840438712789e+00, -2.747165944141573002e-01, -4.768283073662182847e-01, + 0.000000000000000000e+00, 0.000000000000000000e+00, -1.146728572781320565e+00, 0.000000000000000000e+00, + 0.000000000000000000e+00, -1.032915777657712614e-01, 0.000000000000000000e+00, -1.599571851202367112e+00, + -7.161771625083027670e-01, 0.000000000000000000e+00, -6.673724254887420937e-01, 0.000000000000000000e+00, + 0.000000000000000000e+00, 6.565789901111966920e-01, -1.350289421277870661e+00, 0.000000000000000000e+00, + -2.540557809308654491e-01, -2.686275845542446028e+00, 5.361226810123980169e-01, 1.934634164672687645e-02, + 1.299889006228968115e-02, 6.711304002369271604e-01, 1.343899312004804392e+00, 1.279831653805828973e+00, + 5.859059243312456644e-01, 0.000000000000000000e+00, 2.700307766027922884e-01, 2.036695317557343010e+00 +}; + +//bias (1) + w (207) +//const float ndpi_parameters_bd[NUM_PARAMETERS_BD_LOGREG] = { +float ndpi_parameters_bd[NUM_PARAMETERS_BD_LOGREG] = { + -2.953121634313102817e-01, -9.305965891856329863e-05, -1.604178587753208403e-04, -8.663508397764218205e-05, + 3.181501593122275080e-05, 4.869393011205743958e-08, -2.904473357729938132e-09, -1.074435511920153463e-08, + -2.170603991277066491e+00, 6.744305938858414784e-01, 3.953560850413735395e-01, 1.361925254316559641e+00, + 1.157162016392975223e+00, 0.000000000000000000e+00, 5.716702917241568649e+00, 1.141217827469380719e+00, + 1.167390224134238347e-01, 1.735679328274153610e+00, 1.859512740862381497e+00, 4.883258615168795114e-01, + 1.694259125977817693e+00, 0.000000000000000000e+00, 5.554839579235824054e-01, 0.000000000000000000e+00, + 1.345735088930616108e+00, 0.000000000000000000e+00, -2.971613171619579274e-01, 1.047454429359179873e+00, + 9.399973694675579639e-01, 7.598746535296537763e-01, -2.270823795620748431e+00, -1.642785702691181016e-01, + 0.000000000000000000e+00, 0.000000000000000000e+00, 0.000000000000000000e+00, 0.000000000000000000e+00, + -2.238819520252720796e+00, 1.320309301722541573e+00, -1.178170517074989210e+00, -5.248901176332601004e-01, + -1.526086287972392652e+00, -1.448285835686268452e+00, -8.209891928947434803e-01, 0.000000000000000000e+00, + -1.253512931392381846e+00, -2.448957234640304903e-01, -5.824079377648076067e-02, 1.071684992928929603e+00, + -3.143934817584292940e-01, 0.000000000000000000e+00, -1.299533530342349696e-01, -1.253665208636307038e-01, + -3.741714538964039938e-01, 0.000000000000000000e+00, -1.372041577445057836e+00, 0.000000000000000000e+00, + -1.917828430687468666e+00, -1.548156526634417163e-01, 5.069051123254834090e-01, -1.579024137221134161e-01, + -1.048766310256059320e-01, -5.027575687530223547e-01, 0.000000000000000000e+00, -5.343127955429831655e-01, + 0.000000000000000000e+00, 0.000000000000000000e+00, 0.000000000000000000e+00, 5.661515321448501448e-01, + -9.602468971260632591e-01, 1.234839834610549136e+00, 2.568742974036687610e+00, -1.208604146468972962e+00, + 0.000000000000000000e+00, 0.000000000000000000e+00, 0.000000000000000000e+00, 0.000000000000000000e+00, + -1.243861991675339285e+00, 1.460993074682226112e-01, -9.749410166410891199e-01, -4.734754507582912275e-01, + 1.215455435501987813e-01, -2.751667313316082386e-01, -3.432376587556000835e-01, 0.000000000000000000e+00, + 0.000000000000000000e+00, 0.000000000000000000e+00, -3.761783741549818982e-01, 2.086986851763828199e-01, + 0.000000000000000000e+00, 2.649701266176835102e+00, -1.038834923035417024e+00, -1.122672461489231804e-02, + 0.000000000000000000e+00, 0.000000000000000000e+00, 0.000000000000000000e+00, -9.799954015364449322e-01, + -1.991974099640831497e+00, 1.018427492696482473e+00, -4.088041953740855772e-01, 0.000000000000000000e+00, + 0.000000000000000000e+00, 4.487847808661091342e+00, 0.000000000000000000e+00, 0.000000000000000000e+00, + 0.000000000000000000e+00, 0.000000000000000000e+00, 1.697146906018167645e+00, 1.107723995708555842e+00, + -2.057318347123237301e-01, -2.368883723763162974e-01, 4.579837206658370907e-01, -7.570289077756563456e-01, + 8.983818467769307814e-01, 3.537910300939053898e-01, 1.626458397365482922e+00, -3.300836572181266044e-03, + 4.462742143753217761e-02, -1.060184844754213929e-01, 7.810440381838920088e-01, 1.108448216567373246e+00, + 4.128252619360664455e-01, -1.297851442719749060e-01, 0.000000000000000000e+00, 0.000000000000000000e+00, + 0.000000000000000000e+00, 0.000000000000000000e+00, 0.000000000000000000e+00, 5.652218691992449973e-02, + 7.903490726090427465e-01, 8.820592605559850197e-02, 2.826173435847224802e+00, 0.000000000000000000e+00, + 0.000000000000000000e+00, 0.000000000000000000e+00, 0.000000000000000000e+00, 0.000000000000000000e+00, + 0.000000000000000000e+00, -1.865231101602988772e-01, 3.218796325953430237e-01, 0.000000000000000000e+00, + 0.000000000000000000e+00, 1.550652675020544047e+00, 0.000000000000000000e+00, 0.000000000000000000e+00, + 0.000000000000000000e+00, 0.000000000000000000e+00, 0.000000000000000000e+00, 0.000000000000000000e+00, + 0.000000000000000000e+00, -9.513272705900219228e-02, -3.196580534765853243e-01, 0.000000000000000000e+00, + 1.082802500845317706e+00, 0.000000000000000000e+00, 0.000000000000000000e+00, 0.000000000000000000e+00, + 0.000000000000000000e+00, 7.235666749441156398e-01, 6.118502361754621921e-01, -1.185111512789118055e-01, + 0.000000000000000000e+00, 0.000000000000000000e+00, 1.050418002990574778e-01, 1.551405135682879077e+00, + 2.961761913622366293e+00, 1.901323616697461638e+00, 0.000000000000000000e+00, 0.000000000000000000e+00, + -1.332435043211266379e-01, 0.000000000000000000e+00, 0.000000000000000000e+00, 0.000000000000000000e+00, + 0.000000000000000000e+00, 0.000000000000000000e+00, 0.000000000000000000e+00, 0.000000000000000000e+00, + 0.000000000000000000e+00, -1.255550783719393104e+00, 0.000000000000000000e+00, 0.000000000000000000e+00, + 0.000000000000000000e+00, 0.000000000000000000e+00, 0.000000000000000000e+00, 0.000000000000000000e+00, + 0.000000000000000000e+00, 0.000000000000000000e+00, 0.000000000000000000e+00, 0.000000000000000000e+00, + -2.922246847407067860e-01, -5.280391235416594942e-01, 0.000000000000000000e+00, 0.000000000000000000e+00, + 0.000000000000000000e+00, 1.844123585821513034e+00, 0.000000000000000000e+00, 0.000000000000000000e+00, + 0.000000000000000000e+00, -7.628573689172206684e-01, 8.523051946436761561e-01, -5.592366398773165326e-01, + -3.669000025853382807e-01, -5.937559516814655547e-01, 1.445088862911829697e-01, 0.000000000000000000e+00, + 0.000000000000000000e+00, 0.000000000000000000e+00, 0.000000000000000000e+00, -4.041970430267569636e-01, + 2.792928239224993003e+00, -2.814321020845482835e+01, 0.000000000000000000e+00, 0.000000000000000000e+00, + 0.000000000000000000e+00, 3.678601293162953589e-01, 0.000000000000000000e+00, 0.000000000000000000e+00, + 6.131733342900005379e-01, 7.184288961660294515e-01, 0.000000000000000000e+00, 0.000000000000000000e+00, + 0.000000000000000000e+00, 0.000000000000000000e+00, 0.000000000000000000e+00, 0.000000000000000000e+00, + 3.231331452948340566e-01, 0.000000000000000000e+00, 0.000000000000000000e+00, 0.000000000000000000e+00, + 0.000000000000000000e+00, 0.000000000000000000e+00, 0.000000000000000000e+00, 2.409261496800221725e+00, + 0.000000000000000000e+00, 2.548575142888419798e+00, 0.000000000000000000e+00, 0.000000000000000000e+00, + 0.000000000000000000e+00, 0.000000000000000000e+00, 2.458286773678776349e+00, 0.000000000000000000e+00, + 1.319538118247471692e+00, 0.000000000000000000e+00, 0.000000000000000000e+00, 0.000000000000000000e+00, + 0.000000000000000000e+00, 0.000000000000000000e+00, 0.000000000000000000e+00, 0.000000000000000000e+00, + 0.000000000000000000e+00, 0.000000000000000000e+00, 0.000000000000000000e+00, 0.000000000000000000e+00, + 0.000000000000000000e+00, 0.000000000000000000e+00, 0.000000000000000000e+00, 0.000000000000000000e+00, + 0.000000000000000000e+00, 3.743818242393135165e+01, 1.492434857349033628e+01, 0.000000000000000000e+00, + 0.000000000000000000e+00, 6.724128955614088188e-01, 0.000000000000000000e+00, 0.000000000000000000e+00, + 0.000000000000000000e+00, 2.023706156128473044e+00, 9.538479733914937242e+01, 0.000000000000000000e+00, + 0.000000000000000000e+00, 5.004826265911996863e+00, 0.000000000000000000e+00, 0.000000000000000000e+00, + 0.000000000000000000e+00, 0.000000000000000000e+00, 0.000000000000000000e+00, 1.127780548344367917e+00, + 0.000000000000000000e+00, 0.000000000000000000e+00, 0.000000000000000000e+00, 0.000000000000000000e+00, + 6.094069061757222627e+00, 3.151299169326539751e+00, 0.000000000000000000e+00, -2.229793403912785976e+01, + 0.000000000000000000e+00, 0.000000000000000000e+00, 5.949596326773392008e+00, 0.000000000000000000e+00, + 0.000000000000000000e+00, 5.487649125449162391e+00, 0.000000000000000000e+00, 0.000000000000000000e+00, + 0.000000000000000000e+00, 0.000000000000000000e+00, 0.000000000000000000e+00, 0.000000000000000000e+00, + 0.000000000000000000e+00, 3.861348709205134178e+00, 0.000000000000000000e+00, 6.156604990239477715e+00, + 0.000000000000000000e+00, 0.000000000000000000e+00, 0.000000000000000000e+00, 0.000000000000000000e+00, + 0.000000000000000000e+00, 3.517088703524263726e-01, 0.000000000000000000e+00, 1.303045194835739329e+00, + 1.907212085459561379e+01, 3.604016864926741448e+00, 0.000000000000000000e+00, 1.485223477427147998e+00, + 3.537548507508307072e+00, 1.685092396988776331e+00, 0.000000000000000000e+00, 1.545388085903649067e+00, + 6.610815076327216655e-01, 1.796508602929096865e+00, 2.118675147972728823e+00, 9.987341342119526733e-01, + 0.000000000000000000e+00, 2.088903010142080241e+00, 0.000000000000000000e+00, 7.360098931746055229e-01, + 8.749278618310329936e-01, 1.469515615683545828e+00, 4.036900596565609067e-01, 1.907973950826430398e+00, + 1.129753262912140122e-01, 2.098654055515351669e+00, 0.000000000000000000e+00, 0.000000000000000000e+00, + 0.000000000000000000e+00, 0.000000000000000000e+00, 0.000000000000000000e+00, 0.000000000000000000e+00, + 0.000000000000000000e+00, 0.000000000000000000e+00, 0.000000000000000000e+00, 0.000000000000000000e+00, + 0.000000000000000000e+00, 0.000000000000000000e+00, 0.000000000000000000e+00, 0.000000000000000000e+00, + 0.000000000000000000e+00, 0.000000000000000000e+00, 0.000000000000000000e+00, 0.000000000000000000e+00, + 0.000000000000000000e+00, 0.000000000000000000e+00, 0.000000000000000000e+00, 0.000000000000000000e+00, + 0.000000000000000000e+00, 0.000000000000000000e+00, 0.000000000000000000e+00, 0.000000000000000000e+00, + 0.000000000000000000e+00, 0.000000000000000000e+00, 0.000000000000000000e+00, 0.000000000000000000e+00, + 0.000000000000000000e+00, 0.000000000000000000e+00, 0.000000000000000000e+00, 0.000000000000000000e+00, + 0.000000000000000000e+00, 0.000000000000000000e+00, 0.000000000000000000e+00, 0.000000000000000000e+00, + 0.000000000000000000e+00, 0.000000000000000000e+00, 0.000000000000000000e+00, 0.000000000000000000e+00, + 0.000000000000000000e+00, 0.000000000000000000e+00, 0.000000000000000000e+00, 0.000000000000000000e+00, + 0.000000000000000000e+00, 0.000000000000000000e+00, 0.000000000000000000e+00, 0.000000000000000000e+00, + 0.000000000000000000e+00, 0.000000000000000000e+00, 0.000000000000000000e+00, 0.000000000000000000e+00, + 0.000000000000000000e+00, 0.000000000000000000e+00, 0.000000000000000000e+00, 0.000000000000000000e+00, + 0.000000000000000000e+00, 0.000000000000000000e+00, 0.000000000000000000e+00, 0.000000000000000000e+00, + 0.000000000000000000e+00, 0.000000000000000000e+00, 0.000000000000000000e+00, 0.000000000000000000e+00, + 0.000000000000000000e+00, 0.000000000000000000e+00, 0.000000000000000000e+00, 0.000000000000000000e+00, + -4.982390613598663265e+01, 0.000000000000000000e+00, 0.000000000000000000e+00, 0.000000000000000000e+00, + 0.000000000000000000e+00, 0.000000000000000000e+00, 0.000000000000000000e+00, 0.000000000000000000e+00, + 0.000000000000000000e+00, 0.000000000000000000e+00, 0.000000000000000000e+00, 0.000000000000000000e+00, + 0.000000000000000000e+00, 0.000000000000000000e+00, 0.000000000000000000e+00, 0.000000000000000000e+00, + 0.000000000000000000e+00, 0.000000000000000000e+00, 0.000000000000000000e+00, 0.000000000000000000e+00, + 0.000000000000000000e+00, 0.000000000000000000e+00, 0.000000000000000000e+00, 0.000000000000000000e+00, + 0.000000000000000000e+00, 0.000000000000000000e+00, 0.000000000000000000e+00, 0.000000000000000000e+00, + 0.000000000000000000e+00, 0.000000000000000000e+00, 0.000000000000000000e+00, 0.000000000000000000e+00, + 0.000000000000000000e+00, 0.000000000000000000e+00, 0.000000000000000000e+00, 0.000000000000000000e+00, + 0.000000000000000000e+00, 0.000000000000000000e+00, 0.000000000000000000e+00, 0.000000000000000000e+00, + 0.000000000000000000e+00, 0.000000000000000000e+00, 0.000000000000000000e+00, 0.000000000000000000e+00, + 0.000000000000000000e+00, 0.000000000000000000e+00, 0.000000000000000000e+00, 0.000000000000000000e+00, + 0.000000000000000000e+00, 0.000000000000000000e+00, 0.000000000000000000e+00, 0.000000000000000000e+00, + 0.000000000000000000e+00, 0.000000000000000000e+00, 0.000000000000000000e+00, 0.000000000000000000e+00, + 0.000000000000000000e+00, 0.000000000000000000e+00, 0.000000000000000000e+00, 0.000000000000000000e+00, + 0.000000000000000000e+00, 0.000000000000000000e+00, 0.000000000000000000e+00, 0.000000000000000000e+00 +}; + +/** + * \fn void ndpi_merge_splt_arrays (const uint16_t *pkt_len, const struct timeval *pkt_time, + const uint16_t *pkt_len_twin, const struct timeval *pkt_time_twin, + struct timeval start_time, struct timeval start_time_twin, + uint16_t s_idx, uint16_t r_idx, + uint16_t *merged_lens, uint16_t *merged_times, + uint32_t max_num_pkt_len, uint32_t max_merged_num_pkts) + * \param pkt_len length of the packet + * \param pkt_time time of the packet + * \param pkt_len_twin length of the twin packet + * \param pkt_time_twin time of the twin packet + * \param start_time start time + * \param start_time_twin start time of twin + * \param s_idx s index in the merge + * \param r_idx r index in the merge + * \param merged_lens length of the merge + * \param merged_times time of the merge + * \param max_merged_num_pkts number of packets merged + * \return none + */ +void +ndpi_merge_splt_arrays (const uint16_t *pkt_len, const struct timeval *pkt_time, + const uint16_t *pkt_len_twin, const struct timeval *pkt_time_twin, + struct timeval start_time, struct timeval start_time_twin, + uint16_t s_idx, uint16_t r_idx, + uint16_t *merged_lens, uint16_t *merged_times) +{ + int s,r; + struct timeval ts_start = { 0, 0 }; /* initialize to avoid spurious warnings */ + struct timeval tmp, tmp_r; + struct timeval start_m; + + if (r_idx + s_idx == 0) { + return ; + } else if (r_idx == 0) { + ts_start = pkt_time[0]; + tmp = pkt_time[0]; + ndpi_timer_sub(&tmp, &start_time, &start_m); + } else if (s_idx == 0) { + ts_start = pkt_time_twin[0]; + tmp = pkt_time_twin[0]; + ndpi_timer_sub(&tmp, &start_time_twin, &start_m); + } else { + if (ndpi_timer_lt(&start_time, &start_time_twin)) { + ts_start = pkt_time[0]; + tmp = pkt_time[0]; + ndpi_timer_sub(&tmp, &start_time, &start_m); + } else { + // ts_start = pkt_time_twin[0]; + tmp = pkt_time_twin[0]; + ndpi_timer_sub(&tmp, &start_time_twin, &start_m); + } + } + s = r = 0; + while ((s < s_idx) || (r < r_idx)) { + if (s >= s_idx) { + merged_lens[s+r] = pkt_len_twin[r]; + tmp = pkt_time_twin[r]; + ndpi_timer_sub(&tmp, &ts_start, &tmp_r); + merged_times[s+r] = ndpi_timeval_to_milliseconds(tmp_r); + ts_start = tmp; + r++; + } else if (r >= r_idx) { + merged_lens[s+r] = pkt_len[s]; + tmp = pkt_time[s]; + ndpi_timer_sub(&tmp, &ts_start, &tmp_r); + merged_times[s+r] = ndpi_timeval_to_milliseconds(tmp_r); + ts_start = tmp; + s++; + } else { + if (ndpi_timer_lt(&pkt_time[s], &pkt_time_twin[r])) { + merged_lens[s+r] = pkt_len[s]; + tmp = pkt_time[s]; + ndpi_timer_sub(&tmp, &ts_start, &tmp_r); + merged_times[s+r] = ndpi_timeval_to_milliseconds(tmp_r); + ts_start = tmp; + s++; + } else { + merged_lens[s+r] = pkt_len_twin[r]; + tmp = pkt_time_twin[r]; + ndpi_timer_sub(&tmp, &ts_start, &tmp_r); + merged_times[s+r] = ndpi_timeval_to_milliseconds(tmp_r); + ts_start = tmp; + r++; + } + } + } + merged_times[0] = ndpi_timeval_to_milliseconds(start_m); +} + +/* transform lens array to Markov chain */ +static void +ndpi_get_mc_rep_lens (uint16_t *lens, float *length_mc, uint16_t num_packets) +{ + float row_sum; + int prev_packet_size = 0; + int cur_packet_size = 0; + int i, j; + + for (i = 0; i < MC_BINS_LEN*MC_BINS_LEN; i++) { // init to 0 + length_mc[i] = 0.0; + } + + if (num_packets == 0) { + // nothing to do + } else if (num_packets == 1) { + cur_packet_size = (int)min(lens[0]/(float)MC_BIN_SIZE_LEN,(uint16_t)MC_BINS_LEN-1); + length_mc[cur_packet_size + cur_packet_size*MC_BINS_LEN] = 1.0; + } else { + for (i = 1; i < num_packets; i++) { + prev_packet_size = (int)min((uint16_t)(lens[i-1]/(float)MC_BIN_SIZE_LEN),(uint16_t)MC_BINS_LEN-1); + cur_packet_size = (int)min((uint16_t)(lens[i]/(float)MC_BIN_SIZE_LEN),(uint16_t)MC_BINS_LEN-1); + length_mc[prev_packet_size*MC_BINS_LEN + cur_packet_size] += 1.0; + } + // normalize rows of Markov chain + for (i = 0; i < MC_BINS_LEN; i++) { + // find sum + row_sum = 0.0; + for (j = 0; j < MC_BINS_LEN; j++) { + row_sum += length_mc[i*MC_BINS_LEN+j]; + } + if (row_sum != 0.0) { + for (j = 0; j < MC_BINS_LEN; j++) { + length_mc[i*MC_BINS_LEN+j] /= row_sum; + } + } + } + } +} + +/* transform times array to Markov chain */ +void +ndpi_get_mc_rep_times (uint16_t *times, float *time_mc, uint16_t num_packets) +{ + float row_sum; + int prev_packet_time = 0; + int cur_packet_time = 0; + int i, j; + + for (i = 0; i < MC_BINS_TIME*MC_BINS_TIME; i++) { // init to 0 + time_mc[i] = 0.0; + } + if (num_packets == 0) { + // nothing to do + } else if (num_packets == 1) { + cur_packet_time = (int)min(times[0]/(float)MC_BIN_SIZE_TIME,(uint16_t)MC_BINS_TIME-1); + time_mc[cur_packet_time + cur_packet_time*MC_BINS_TIME] = 1.0; + } else { + for (i = 1; i < num_packets; i++) { + prev_packet_time = (int)min((uint16_t)(times[i-1]/(float)MC_BIN_SIZE_TIME),(uint16_t)MC_BINS_TIME-1); + cur_packet_time = (int)min((uint16_t)(times[i]/(float)MC_BIN_SIZE_TIME),(uint16_t)MC_BINS_TIME-1); + time_mc[prev_packet_time*MC_BINS_TIME + cur_packet_time] += 1.0; + } + // normalize rows of Markov chain + for (i = 0; i < MC_BINS_TIME; i++) { + // find sum + row_sum = 0.0; + for (j = 0; j < MC_BINS_TIME; j++) { + row_sum += time_mc[i*MC_BINS_TIME+j]; + } + if (row_sum != 0.0) { + for (j = 0; j < MC_BINS_TIME; j++) { + time_mc[i*MC_BINS_TIME+j] /= row_sum; + } + } + } + } +} + +/** + * \fn float classify (const unsigned short *pkt_len, const struct timeval *pkt_time, + const unsigned short *pkt_len_twin, const struct timeval *pkt_time_twin, + struct timeval start_time, struct timeval start_time_twin, uint32_t max_num_pkt_len, + uint16_t sp, uint16_t dp, uint32_t op, uint32_t ip, uint32_t np_o, uint32_t np_i, + uint32_t ob, uint32_t ib, uint16_t use_bd, const uint32_t *bd, const uint32_t *bd_t) + * \param pkt_len length of the packet + * \param pkt_time time of the packet + * \param pkt_len_twin length of the packet twin + * \param pkt_time_twin time of the packet twin + * \param start_time start time + * \param start_time_twin start time of the twin + * \param max_num_pkt_len maximum len of number of packets + * \param sp + * \param dp + * \param op + * \param ip + * \param np_o + * \param np_i + * \param ob + * \param ib + * \param use_bd + * \param *bd pointer to bd + * \param *bd_t pointer to bd type + * \return float score + */ +float +ndpi_classify (const unsigned short *pkt_len, const struct timeval *pkt_time, + const unsigned short *pkt_len_twin, const struct timeval *pkt_time_twin, + struct timeval start_time, struct timeval start_time_twin, uint32_t max_num_pkt_len, + uint16_t sp, uint16_t dp, uint32_t op, uint32_t ip, uint32_t np_o, uint32_t np_i, + uint32_t ob, uint32_t ib, uint16_t use_bd, const uint32_t *bd, const uint32_t *bd_t) +{ + + float features[NUM_PARAMETERS_BD_LOGREG] = {1.0}; + float mc_lens[MC_BINS_LEN*MC_BINS_LEN]; + float mc_times[MC_BINS_TIME*MC_BINS_TIME]; + uint32_t i; + float score = 0.0; + + uint32_t op_n = min(np_o, max_num_pkt_len); + uint32_t ip_n = min(np_i, max_num_pkt_len); + uint16_t *merged_lens = NULL; + uint16_t *merged_times = NULL; + + for (i = 1; i < NUM_PARAMETERS_BD_LOGREG; i++) { + features[i] = 0.0; + } + + merged_lens = calloc(1, sizeof(uint16_t)*(op_n + ip_n)); + merged_times = calloc(1, sizeof(uint16_t)*(op_n + ip_n)); + if (!merged_lens || !merged_times) { + free(merged_lens); + free(merged_times); + return(score); + } + + // fill out meta data + features[1] = (float)dp; // destination port + features[2] = (float)sp; // source port + features[3] = (float)ip; // inbound packets + features[4] = (float)op; // outbound packets + features[5] = (float)ib; // inbound bytes + features[6] = (float)ob; // outbound bytes + features[7] = 0.0;// skipping 7 until we process the pkt_time arrays + + // find the raw features + ndpi_merge_splt_arrays(pkt_len, pkt_time, pkt_len_twin, pkt_time_twin, start_time, start_time_twin, op_n, ip_n, + merged_lens, merged_times); + + // find new duration + for (i = 0; i < op_n+ip_n; i++) { + features[7] += (float)merged_times[i]; + } + + // get the Markov chain representation for the lengths + ndpi_get_mc_rep_lens(merged_lens, mc_lens, op_n+ip_n); + + // get the Markov chain representation for the times + ndpi_get_mc_rep_times(merged_times, mc_times, op_n+ip_n); + + // fill out lens/times in feature vector + for (i = 0; i < MC_BINS_LEN*MC_BINS_LEN; i++) { + features[i+8] = mc_lens[i]; // lengths + } + for (i = 0; i < MC_BINS_TIME*MC_BINS_TIME; i++) { + features[i+8+MC_BINS_LEN*MC_BINS_LEN] = mc_times[i]; // times + } + + // fill out byte distribution features + if (ob+ib > 100 && use_bd) { + for (i = 0; i < NUM_BD_VALUES; i++) { + if (pkt_len_twin != NULL) { + features[i+8+MC_BINS_LEN*MC_BINS_LEN+MC_BINS_TIME*MC_BINS_TIME] = (bd[i]+bd_t[i])/((float)(ob+ib)); + } else { + features[i+8+MC_BINS_LEN*MC_BINS_LEN+MC_BINS_TIME*MC_BINS_TIME] = bd[i]/((float)(ob)); + } + } + } + + if (ob+ib > 100 && use_bd) { + score = ndpi_parameters_bd[0]; + for (i = 1; i < NUM_PARAMETERS_BD_LOGREG; i++) { + score += features[i]*ndpi_parameters_bd[i]; + } + } else { + for (i = 0; i < NUM_PARAMETERS_SPLT_LOGREG; i++) { + score += features[i]*ndpi_parameters_splt[i]; + } + } + + score = min(-score,500.0); // check b/c overflow + + free(merged_lens); + free(merged_times); + + return 1.0/(1.0+exp(score)); +} + +/** + * \fn void update_params (char *splt_params, char *bd_params) + * \brief if a user supplies new parameter files, update parameters splt/bd + * \param param_type type of new parameters to update + * \param params file name with new parameters + * \reutrn none + */ +void +ndpi_update_params (classifier_type_codes_t param_type, const char *param_file) +{ + float param; + FILE *fp; + int count = 0; + + switch (param_type) { + case (SPLT_PARAM_TYPE): + count = 0; + fp = fopen(param_file,"r"); + if (fp != NULL) { + while (fscanf(fp, "%f", ¶m) != EOF) { + ndpi_parameters_splt[count] = param; + count++; + if (count >= NUM_PARAMETERS_SPLT_LOGREG) { + break; + } + } + fclose(fp); + } + break; + + case (BD_PARAM_TYPE): + count = 0; + fp = fopen(param_file,"r"); + if (fp != NULL) { + while (fscanf(fp, "%f", ¶m) != EOF) { + ndpi_parameters_bd[count] = param; + count++; + if (count >= NUM_PARAMETERS_BD_LOGREG) { + break; + } + } + fclose(fp); + } + break; + + default: + printf("error: unknown paramerter type (%d)", param_type); + break; + } +} + +/* ********************************************************************* + * --------------------------------------------------------------------- + * Time functions + * For portability and static analysis, we define our own timer + * comparison functions (rather than use non-standard + * timercmp/timersub macros) + * --------------------------------------------------------------------- + * ********************************************************************* + */ + +/** + * \brief Compare two times to see if they are equal + * \param a First time value + * \param b Second time value + * \return 1 if equal, 0 otherwise + */ +unsigned int +ndpi_timer_eq(const struct timeval *a, + const struct timeval *b) +{ + if (a->tv_sec == b->tv_sec && a->tv_usec == b->tv_usec) { + return 1; + } + + return 0; +} + +unsigned int +ndpi_timer_lt(const struct timeval *a, + const struct timeval *b) +{ + return (a->tv_sec == b->tv_sec) ? + (a->tv_usec < b->tv_usec):(a->tv_sec < b->tv_sec); +} + +/** + * \brief Calculate the difference betwen two times (result = a - b) + * \param a First time value + * \param b Second time value + * \param result The difference between the two time values + * \return none + */ +void +ndpi_timer_sub(const struct timeval *a, + const struct timeval *b, + struct timeval *result) +{ + result->tv_sec = a->tv_sec - b->tv_sec; + result->tv_usec = a->tv_usec - b->tv_usec; + if (result->tv_usec < 0) { + --result->tv_sec; + result->tv_usec += 1000000; + } +} + +/** + * \brief Zeroize a timeval. + * \param a Timeval to zero out + * \return none + */ +void +ndpi_timer_clear(struct timeval *a) +{ + a->tv_sec = a->tv_usec = 0; +} + +/** + * \brief Calculate the milliseconds representation of a timeval. + * \param ts Timeval + * \return unsigned int - Milliseconds + */ +unsigned int +ndpi_timeval_to_milliseconds(struct timeval ts) +{ + unsigned int result = ts.tv_usec / 1000 + ts.tv_sec * 1000; + return result; +} + +void +ndpi_log_timestamp(char *log_ts, u_int log_ts_len) +{ + struct timeval tv; + time_t nowtime; + struct tm nowtm_r; + char tmbuf[NDPI_TIMESTAMP_LEN]; + + gettimeofday(&tv, NULL); + nowtime = tv.tv_sec; + localtime_r(&nowtime, &nowtm_r); + strftime(tmbuf, NDPI_TIMESTAMP_LEN, "%H:%M:%S", &nowtm_r); + snprintf(log_ts, log_ts_len, "%s.%06ld", tmbuf, (long)tv.tv_usec); +} diff --git a/src/lib/ndpi_content_match.c.inc b/src/lib/ndpi_content_match.c.inc index 0d8e1769a..9c0356ec8 100644 --- a/src/lib/ndpi_content_match.c.inc +++ b/src/lib/ndpi_content_match.c.inc @@ -416,6 +416,7 @@ static ndpi_network host_protocol_list[] = { { 0x08129000 /* 8.18.144.0/24 */, 24, NDPI_PROTOCOL_AMAZON }, { 0x08129100 /* 8.18.145.0/24 */, 24, NDPI_PROTOCOL_AMAZON }, + { 0x08F84F00 /* 8.248.79.0/24 */, 24, NDPI_PROTOCOL_AMAZON_VIDEO }, { 0x0D200000 /* 13.32.0.0/14 */, 14, NDPI_PROTOCOL_AMAZON }, { 0x0D340000 /* 13.52.0.0/14 */, 14, NDPI_PROTOCOL_AMAZON }, { 0x0D380000 /* 13.56.0.0/14 */, 14, NDPI_PROTOCOL_AMAZON }, @@ -489,6 +490,7 @@ static ndpi_network host_protocol_list[] = { { 0x34520000 /* 52.82.0.0/14 */, 14, NDPI_PROTOCOL_AMAZON }, { 0x34580000 /* 52.88.0.0/13 */, 13, NDPI_PROTOCOL_AMAZON }, { 0x345A0000 /* 52.90.0.0/15 */, 15, NDPI_PROTOCOL_AMAZON }, + { 0x345EE000 /* 52.94.224.0/19 */, 19, NDPI_PROTOCOL_AMAZON }, { 0x345F0000 /* 52.95.0.0/21 */, 21, NDPI_PROTOCOL_AMAZON }, { 0x345F0A00 /* 52.95.10.0/23 */, 23, NDPI_PROTOCOL_AMAZON }, { 0x345F0C00 /* 52.95.12.0/22 */, 22, NDPI_PROTOCOL_AMAZON }, @@ -8296,12 +8298,18 @@ ndpi_protocol_match host_match[] = { { "d25xi40x97liuc.cloudfront.net", NULL, "d25xi40x97liuc\\.cloudfront\\.net", "AmazonVideo", NDPI_PROTOCOL_AMAZON_VIDEO, NDPI_PROTOCOL_CATEGORY_VIDEO, NDPI_PROTOCOL_FUN }, { ".aiv-delivery.net", NULL, "\\.aiv-delivery\\.net", "AmazonVideo", NDPI_PROTOCOL_AMAZON_VIDEO, NDPI_PROTOCOL_CATEGORY_VIDEO, NDPI_PROTOCOL_FUN }, { ".aiv-cdn.net", NULL, "\\.aiv-cdn\\.net", "AmazonVideo", NDPI_PROTOCOL_AMAZON_VIDEO, NDPI_PROTOCOL_CATEGORY_VIDEO, NDPI_PROTOCOL_FUN }, + { "1s3.lvlt.dash.us.aiv-cdn.net.c.footprint.net", NULL, "1s3\\.lvlt\\.dash\\.us\\.aiv-cdn\\.net\\.c\\.footprint\\.net", "AmazonVideo", NDPI_PROTOCOL_AMAZON_VIDEO, NDPI_PROTOCOL_CATEGORY_STREAMING, NDPI_PROTOCOL_FUN }, + { ".cloudfront.net", NULL, "\\.cloudfront\\.net", "AmazonVideo", NDPI_PROTOCOL_AMAZON_VIDEO, NDPI_PROTOCOL_CATEGORY_VIDEO, NDPI_PROTOCOL_FUN }, + { ".s.loris.llnwd.net", NULL, "\\.s\\.loris\\.llnwd\\.net", "AmazonVideo", NDPI_PROTOCOL_AMAZON_VIDEO, NDPI_PROTOCOL_CATEGORY_VIDEO, NDPI_PROTOCOL_FUN }, + { "atv-ext.amazon.com", NULL, NULL, "AmazonVideo", NDPI_PROTOCOL_AMAZON_VIDEO, NDPI_PROTOCOL_CATEGORY_VIDEO, NDPI_PROTOCOL_FUN }, + { "c.media-amazon.com", NULL, NULL, "AmazonVideo", NDPI_PROTOCOL_AMAZON_VIDEO, NDPI_PROTOCOL_CATEGORY_VIDEO, NDPI_PROTOCOL_FUN }, { "amazon.", NULL, NULL, "Amazon", NDPI_PROTOCOL_AMAZON, NDPI_PROTOCOL_CATEGORY_WEB, NDPI_PROTOCOL_ACCEPTABLE }, { "amazon.com", NULL, "amazon" TLD, "Amazon", NDPI_PROTOCOL_AMAZON, NDPI_PROTOCOL_CATEGORY_WEB, NDPI_PROTOCOL_ACCEPTABLE }, { "images-amazon.com", NULL, "images-amazon" TLD, "Amazon", NDPI_PROTOCOL_AMAZON, NDPI_PROTOCOL_CATEGORY_WEB, NDPI_PROTOCOL_ACCEPTABLE }, { "amazonaws.com", NULL, "amazonaws" TLD, "Amazon", NDPI_PROTOCOL_AMAZON, NDPI_PROTOCOL_CATEGORY_WEB, NDPI_PROTOCOL_ACCEPTABLE }, { "amazon-adsystem.com", NULL, "amazon-adsystem" TLD, "Amazon", NDPI_PROTOCOL_AMAZON, NDPI_PROTOCOL_CATEGORY_WEB, NDPI_PROTOCOL_ACCEPTABLE }, { ".cloudfront.net", NULL, "\\.cloudfront" TLD, "Amazon", NDPI_PROTOCOL_AMAZON, NDPI_PROTOCOL_CATEGORY_WEB, NDPI_PROTOCOL_ACCEPTABLE }, + { ".us-west-2.compute.amazonaws.com", NULL, "\\.us-west-2\\.compute\\.amazonaws\\.com", "Amazon", NDPI_PROTOCOL_AMAZON, NDPI_PROTOCOL_CATEGORY_WEB, NDPI_PROTOCOL_ACCEPTABLE }, { ".push.apple.com", NULL, "\\.push\\.apple" TLD, "ApplePush", NDPI_PROTOCOL_APPLE_PUSH, NDPI_PROTOCOL_CATEGORY_CLOUD, NDPI_PROTOCOL_SAFE }, { ".apple-dns.net", NULL, "\\.apple-dns" TLD, "Apple", NDPI_PROTOCOL_APPLE, NDPI_PROTOCOL_CATEGORY_WEB, NDPI_PROTOCOL_SAFE }, @@ -8320,7 +8328,22 @@ ndpi_protocol_match host_match[] = { { "itunes-apple.com", NULL, "itunes-apple" TLD, "AppleStore", NDPI_PROTOCOL_APPLESTORE, NDPI_PROTOCOL_CATEGORY_SW_UPDATE, NDPI_PROTOCOL_SAFE }, { "itunes.apple.com", NULL, "itunes\\.apple" TLD, "AppleiTunes", NDPI_PROTOCOL_APPLE_ITUNES, NDPI_PROTOCOL_CATEGORY_STREAMING, NDPI_PROTOCOL_FUN }, { "tlnk.io", NULL, "tlnk" TLD, "AppleiTunes", NDPI_PROTOCOL_APPLE_ITUNES, NDPI_PROTOCOL_CATEGORY_STREAMING, NDPI_PROTOCOL_FUN }, - + { ".wbagora.com", NULL, "wbagora" TLD, "Playstation", NDPI_PROTOCOL_PLAYSTATION, NDPI_PROTOCOL_CATEGORY_GAME, NDPI_PROTOCOL_UNRATED }, + { ".wbplay.com", NULL, "wbplay" TLD, "Playstation", NDPI_PROTOCOL_PLAYSTATION, NDPI_PROTOCOL_CATEGORY_GAME, NDPI_PROTOCOL_UNRATED }, + { ".xbox.com", NULL, "xbox" TLD, "Xbox", NDPI_PROTOCOL_XBOX, NDPI_PROTOCOL_CATEGORY_GAME, NDPI_PROTOCOL_FUN }, + { ".xboxlive.com", NULL, "xboxlive" TLD, "Xbox", NDPI_PROTOCOL_XBOX, NDPI_PROTOCOL_CATEGORY_GAME, NDPI_PROTOCOL_FUN }, + { ".xboxlive.com.akadns.net", NULL, "xboxlive" TLD, "Xbox", NDPI_PROTOCOL_XBOX, NDPI_PROTOCOL_CATEGORY_GAME, NDPI_PROTOCOL_FUN }, + { ".xboxlive.com.c.footprint.net", NULL, "xboxlive" TLD, "Xbox", NDPI_PROTOCOL_XBOX, NDPI_PROTOCOL_CATEGORY_GAME, NDPI_PROTOCOL_FUN }, + { ".edgecastcdn.net", NULL, "egdecastcdn" TLD, "Unknown", NDPI_PROTOCOL_GENERIC, NDPI_PROTOCOL_CATEGORY_MEDIA, NDPI_PROTOCOL_SAFE }, + { ".hwcdn.net", NULL, "hwcdn" TLD, "Unknown", NDPI_PROTOCOL_GENERIC, NDPI_PROTOCOL_CATEGORY_MEDIA, NDPI_PROTOCOL_SAFE }, + { ".llnwd.net", NULL, "llnwd" TLD, "Unknown", NDPI_PROTOCOL_GENERIC, NDPI_PROTOCOL_CATEGORY_MEDIA, NDPI_PROTOCOL_SAFE }, + { ".llns.net", NULL, "llns" TLD, "Unknown", NDPI_PROTOCOL_GENERIC, NDPI_PROTOCOL_CATEGORY_MEDIA, NDPI_PROTOCOL_SAFE }, + { ".fastly.net", NULL, "fastly" TLD, "Unknown", NDPI_PROTOCOL_GENERIC, NDPI_PROTOCOL_CATEGORY_MEDIA, NDPI_PROTOCOL_SAFE }, + { ".akamaiedge.net", NULL, "akamaiedge" TLD, "Unknown", NDPI_PROTOCOL_GENERIC, NDPI_PROTOCOL_CATEGORY_MEDIA, NDPI_PROTOCOL_SAFE }, + { "e13555.b.akamaiedge.net", NULL, "e13555\\.b\\.akamaiedge" TLD, "Playstation", NDPI_PROTOCOL_PLAYSTATION, NDPI_PROTOCOL_CATEGORY_GAME, NDPI_PROTOCOL_FUN }, + { "e1800.d.akamaiedge.net", NULL, "e1800\\.d\\.akamaiedge" TLD, "Playstation", NDPI_PROTOCOL_PLAYSTATION, NDPI_PROTOCOL_CATEGORY_GAME, NDPI_PROTOCOL_FUN }, + { "e1879.e7.akamaiedge.net", NULL, "e1879\\.e7\\.akamaiedge" TLD, "Playstation", NDPI_PROTOCOL_PLAYSTATION, NDPI_PROTOCOL_CATEGORY_GAME, NDPI_PROTOCOL_FUN }, + { ".vultr.com", NULL, "vultr" TLD, "Unknown", NDPI_PROTOCOL_GENERIC, NDPI_PROTOCOL_CATEGORY_CLOUD, NDPI_PROTOCOL_ACCEPTABLE }, { ".cnn.c", NULL, "\\.cnn" TLD, "CNN", NDPI_PROTOCOL_CNN, NDPI_PROTOCOL_CATEGORY_WEB, NDPI_PROTOCOL_SAFE }, { ".cnn.net", NULL, NULL, "CNN", NDPI_PROTOCOL_CNN, NDPI_PROTOCOL_CATEGORY_WEB, NDPI_PROTOCOL_SAFE }, @@ -8479,6 +8502,9 @@ ndpi_protocol_match host_match[] = { { "mmg-fna.whatsapp.net", NULL, "mmg-fna\\.whatsapp" TLD, "WhatsAppFiles", NDPI_PROTOCOL_WHATSAPP_FILES, NDPI_PROTOCOL_CATEGORY_DOWNLOAD_FT, NDPI_PROTOCOL_ACCEPTABLE }, { ".whatsapp.", NULL, "\\.whatsapp" TLD, "WhatsApp", NDPI_PROTOCOL_WHATSAPP, NDPI_PROTOCOL_CATEGORY_CHAT, NDPI_PROTOCOL_ACCEPTABLE }, + { "g.whatsapp.net", NULL, "g\\.whatsapp" TLD, "WhatsApp", NDPI_PROTOCOL_WHATSAPP, NDPI_PROTOCOL_CATEGORY_CHAT, NDPI_PROTOCOL_ACCEPTABLE }, + { "v.whatsapp.net", NULL, "v\\.whatsapp" TLD, "WhatsApp", NDPI_PROTOCOL_WHATSAPP, NDPI_PROTOCOL_CATEGORY_CHAT, NDPI_PROTOCOL_ACCEPTABLE }, + { "mmg.whatsapp.net", NULL, "mmg\\.whatsapp" TLD, "WhatsApp", NDPI_PROTOCOL_WHATSAPP, NDPI_PROTOCOL_CATEGORY_CHAT, NDPI_PROTOCOL_ACCEPTABLE }, { ".yahoo.", NULL, "\\.yahoo" TLD, "Yahoo", NDPI_PROTOCOL_YAHOO, NDPI_PROTOCOL_CATEGORY_WEB, NDPI_PROTOCOL_SAFE }, { ".yimg.com", NULL, "\\.yimg" TLD, "Yahoo", NDPI_PROTOCOL_YAHOO, NDPI_PROTOCOL_CATEGORY_WEB, NDPI_PROTOCOL_SAFE }, diff --git a/src/lib/ndpi_main.c b/src/lib/ndpi_main.c index 266e7e816..9b6bdc632 100644 --- a/src/lib/ndpi_main.c +++ b/src/lib/ndpi_main.c @@ -1019,8 +1019,13 @@ static void ndpi_init_protocol_defaults(struct ndpi_detection_module_struct *ndp ndpi_set_proto_defaults(ndpi_mod, NDPI_PROTOCOL_FUN, NDPI_PROTOCOL_XBOX, 0 /* can_have_a_subprotocol */, no_master, no_master, "Xbox", NDPI_PROTOCOL_CATEGORY_GAME, - ndpi_build_default_ports(ports_a, 0, 0, 0, 0, 0) /* TCP */, - ndpi_build_default_ports(ports_b, 0, 0, 0, 0, 0) /* UDP */); + ndpi_build_default_ports(ports_a, 3074, 3076, 0, 0, 0) /* TCP */, + ndpi_build_default_ports(ports_b, 3074, 3076, 500, 3544, 4500) /* UDP */); + ndpi_set_proto_defaults(ndpi_mod, NDPI_PROTOCOL_FUN, NDPI_PROTOCOL_PLAYSTATION, + 0 /* can_have_a_subprotocol */, no_master, + no_master, "Playstation", NDPI_PROTOCOL_CATEGORY_GAME, + ndpi_build_default_ports(ports_a, 1935, 3478, 3479, 3480, 0) /* TCP */, + ndpi_build_default_ports(ports_b, 3478, 3479, 0, 0, 0) /* UDP */); ndpi_set_proto_defaults(ndpi_mod, NDPI_PROTOCOL_FUN, NDPI_PROTOCOL_QQ, 0 /* can_have_a_subprotocol */, no_master, no_master, "QQ", NDPI_PROTOCOL_CATEGORY_CHAT, @@ -1661,7 +1666,7 @@ static void ndpi_init_protocol_defaults(struct ndpi_detection_module_struct *ndp ndpi_build_default_ports(ports_b, 0, 0, 0, 0, 0)); /* UDP */ ndpi_set_proto_defaults(ndpi_mod, NDPI_PROTOCOL_ACCEPTABLE, NDPI_PROTOCOL_HANGOUT_DUO, 0 /* can_have_a_subprotocol */, no_master, - no_master, "GoogleHangout", NDPI_PROTOCOL_CATEGORY_VOIP, + no_master, "GoogleHangoutDuo", NDPI_PROTOCOL_CATEGORY_VOIP, ndpi_build_default_ports(ports_a, 0, 0, 0, 0, 0) /* TCP */, ndpi_build_default_ports(ports_b, 0, 0, 0, 0, 0) /* UDP */); ndpi_set_proto_defaults(ndpi_mod, NDPI_PROTOCOL_ACCEPTABLE, NDPI_PROTOCOL_BJNP, @@ -1714,6 +1719,16 @@ static void ndpi_init_protocol_defaults(struct ndpi_detection_module_struct *ndp no_master, "AJP", NDPI_PROTOCOL_CATEGORY_WEB, ndpi_build_default_ports(ports_a, 8009, 0, 0, 0, 0) /* TCP */, ndpi_build_default_ports(ports_b, 0, 0, 0, 0, 0) /* UDP */); + ndpi_set_proto_defaults(ndpi_mod, NDPI_PROTOCOL_ACCEPTABLE, NDPI_PROTOCOL_TARGUS_GETDATA, + 0 /* can_have_a_subprotocol */, no_master, + no_master, "Targus Dataspeed", NDPI_PROTOCOL_CATEGORY_NETWORK, + ndpi_build_default_ports(ports_a, 5001, 5201, 0, 0, 0) /* TCP */, + ndpi_build_default_ports(ports_b, 5001, 5201, 0, 0, 0) /* UDP */); + ndpi_set_proto_defaults(ndpi_mod, NDPI_PROTOCOL_ACCEPTABLE, NDPI_PROTOCOL_AMAZON_VIDEO, + 0 /* can_have_a_subprotocol */, no_master, + no_master, "AmazonVideo", NDPI_PROTOCOL_CATEGORY_CLOUD, + ndpi_build_default_ports(ports_a, 0, 0, 0, 0, 0) /* TCP */, + ndpi_build_default_ports(ports_b, 0, 0, 0, 0, 0) /* UDP */); /* calling function for host and content matched protocols */ init_string_based_protocols(ndpi_mod); @@ -3256,6 +3271,12 @@ void ndpi_set_protocol_detection_bitmask2(struct ndpi_detection_module_struct *n /* WireGuard VPN */ init_wireguard_dissector(ndpi_struct, &a, detection_bitmask); + /* AMAZON_VIDEO */ + init_amazon_video_dissector(ndpi_struct, &a, detection_bitmask); + + /* Targus Getdata */ + init_targus_getdata_dissector(ndpi_struct, &a, detection_bitmask); + /* ----------------------------------------------------------------- */ ndpi_struct->callback_buffer_size = a; @@ -4223,7 +4244,6 @@ int ndpi_load_hostname_category(struct ndpi_detection_module_struct *ndpi_struct struct hs_list *h = (struct hs_list*)malloc(sizeof(struct hs_list)); if(h) { - int i, j; h->expression = ndpi_strdup(name), h->id = (unsigned int)category; if(h->expression == NULL) { diff --git a/src/lib/ndpi_utils.c b/src/lib/ndpi_utils.c index 992eaa297..8959aa05e 100644 --- a/src/lib/ndpi_utils.c +++ b/src/lib/ndpi_utils.c @@ -916,6 +916,14 @@ static void ndpi_deserialize_single_uint32(ndpi_serializer *deserializer, /* ********************************** */ +static void ndpi_deserialize_single_int32(ndpi_serializer *deserializer, + int32_t *s) { + *s = ntohl(*((int32_t *) &deserializer->buffer[deserializer->size_used])); + deserializer->size_used += sizeof(int32_t); +} + +/* ********************************** */ + static void ndpi_deserialize_single_uint64(ndpi_serializer *deserializer, u_int64_t *s) { *s = ndpi_ntohll(*(u_int64_t*)&deserializer->buffer[deserializer->size_used]); @@ -924,6 +932,14 @@ static void ndpi_deserialize_single_uint64(ndpi_serializer *deserializer, /* ********************************** */ +static void ndpi_deserialize_single_int64(ndpi_serializer *deserializer, + int64_t *s) { + *s = ndpi_ntohll(*(int64_t*)&deserializer->buffer[deserializer->size_used]); + deserializer->size_used += sizeof(int64_t); +} + +/* ********************************** */ + static void ndpi_deserialize_single_string(ndpi_serializer *deserializer, ndpi_string *v) { v->str_len = ntohs(*((u_int16_t *) &deserializer->buffer[deserializer->size_used])); @@ -1100,6 +1116,84 @@ int ndpi_serialize_uint32_string(ndpi_serializer *serializer, /* ********************************** */ +int ndpi_serialize_string_int32(ndpi_serializer *serializer, + const char *key, int32_t value) { + u_int16_t klen = strlen(key); + u_int32_t buff_diff = serializer->buffer_size - serializer->size_used; + u_int32_t needed = + sizeof(u_int8_t) /* type */ + + sizeof(u_int16_t) /* key len */ + + klen /* key */ + + sizeof(u_int32_t); + + if(serializer->fmt == ndpi_serialization_format_json) + needed += 16 + klen; + + if(buff_diff < needed) { + if (ndpi_extend_serializer_buffer(serializer, needed - buff_diff) < 0) + return(-1); + buff_diff = serializer->buffer_size - serializer->size_used; + } + + if(serializer->fmt == ndpi_serialization_format_json) { + ndpi_serialize_json_pre(serializer); + serializer->size_used += ndpi_json_string_escape(key, klen, + (char *) &serializer->buffer[serializer->size_used], buff_diff); + buff_diff = serializer->buffer_size - serializer->size_used; + serializer->size_used += snprintf((char *) &serializer->buffer[serializer->size_used], buff_diff, + ":%d", value); + ndpi_serialize_json_post(serializer); + } else { + serializer->buffer[serializer->size_used++] = ndpi_serialization_string_int32; + + ndpi_serialize_single_string(serializer, key, klen); + ndpi_serialize_single_uint32(serializer, value); + } + + return(0); +} + +/* ********************************** */ + +int ndpi_serialize_string_int64(ndpi_serializer *serializer, + const char *key, int64_t value) { + u_int16_t klen = strlen(key); + u_int32_t buff_diff = serializer->buffer_size - serializer->size_used; + u_int32_t needed = + sizeof(u_int8_t) /* type */ + + sizeof(u_int16_t) /* key len */ + + klen /* key */ + + sizeof(u_int32_t); + + if(serializer->fmt == ndpi_serialization_format_json) + needed += 16 + klen; + + if(buff_diff < needed) { + if (ndpi_extend_serializer_buffer(serializer, needed - buff_diff) < 0) + return(-1); + buff_diff = serializer->buffer_size - serializer->size_used; + } + + if(serializer->fmt == ndpi_serialization_format_json) { + ndpi_serialize_json_pre(serializer); + serializer->size_used += ndpi_json_string_escape(key, klen, + (char *) &serializer->buffer[serializer->size_used], buff_diff); + buff_diff = serializer->buffer_size - serializer->size_used; + serializer->size_used += snprintf((char *) &serializer->buffer[serializer->size_used], buff_diff, + ":%lld", value); + ndpi_serialize_json_post(serializer); + } else { + serializer->buffer[serializer->size_used++] = ndpi_serialization_string_int64; + + ndpi_serialize_single_string(serializer, key, klen); + ndpi_serialize_single_uint32(serializer, value); + } + + return(0); +} + +/* ********************************** */ + int ndpi_serialize_string_uint32(ndpi_serializer *serializer, const char *key, u_int32_t value) { u_int16_t klen = strlen(key); @@ -1347,6 +1441,50 @@ int ndpi_deserialize_uint32_string(ndpi_deserializer *deserializer, /* ********************************** */ +int ndpi_deserialize_string_int32(ndpi_deserializer *deserializer, + ndpi_string *key, int32_t *value) { + if(ndpi_deserialize_get_nextitem_type(deserializer) == ndpi_serialization_string_int32) { + u_int32_t buff_diff = deserializer->buffer_size - deserializer->size_used; + u_int32_t expected = + sizeof(u_int8_t) /* type */ + + sizeof(u_int16_t) /* key len */ + + sizeof(int32_t); + + if(buff_diff < expected) return(-2); + + deserializer->size_used++; /* Skip element type */ + ndpi_deserialize_single_string(deserializer, key); + ndpi_deserialize_single_int32(deserializer, value); + + return(0); + } else + return(-1); +} + +/* ********************************** */ + +int ndpi_deserialize_string_int64(ndpi_deserializer *deserializer, + ndpi_string *key, int64_t *value) { + if(ndpi_deserialize_get_nextitem_type(deserializer) == ndpi_serialization_string_int64) { + u_int32_t buff_diff = deserializer->buffer_size - deserializer->size_used; + u_int32_t expected = + sizeof(u_int8_t) /* type */ + + sizeof(u_int16_t) /* key len */ + + sizeof(int64_t); + + if(buff_diff < expected) return(-2); + + deserializer->size_used++; /* Skip element type */ + ndpi_deserialize_single_string(deserializer, key); + ndpi_deserialize_single_int64(deserializer, value); + + return(0); + } else + return(-1); +} + +/* ********************************** */ + int ndpi_deserialize_string_uint32(ndpi_deserializer *deserializer, ndpi_string *key, u_int32_t *value) { if(ndpi_deserialize_get_nextitem_type(deserializer) == ndpi_serialization_string_uint32) { diff --git a/src/lib/protocols/amazon_video.c b/src/lib/protocols/amazon_video.c new file mode 100644 index 000000000..41356d9ad --- /dev/null +++ b/src/lib/protocols/amazon_video.c @@ -0,0 +1,81 @@ +/* + * amazon_video.c + * + * Copyright (C) 2018 by ntop.org + * + * This file is part of nDPI, an open source deep packet inspection + * library based on the OpenDPI and PACE technology by ipoque GmbH + * + * nDPI is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * nDPI is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with nDPI. If not, see <http://www.gnu.org/licenses/>. + * + */ + +#include "ndpi_protocol_ids.h" + +#define NDPI_CURRENT_PROTO NDPI_PROTOCOL_AMAZON_VIDEO + +#include "ndpi_api.h" + +static void ndpi_check_amazon_video(struct ndpi_detection_module_struct *ndpi_struct, + struct ndpi_flow_struct *flow) { + + struct ndpi_packet_struct *packet = &flow->packet; + + NDPI_LOG_DBG(ndpi_struct, "search Amazon Prime\n"); + + if((packet->tcp != NULL) && + (packet->payload[0] == 0xFE && + packet->payload[1] == 0xED && + packet->payload[2] == 0xFA && + packet->payload[3] == 0xCE)) + { + NDPI_LOG_INFO(ndpi_struct, "found Amazon Video on TCP\n"); + ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_AMAZON_VIDEO, NDPI_PROTOCOL_UNKNOWN); + return; + } + else if((packet->udp != NULL) && + (packet->payload[0] == 0xDE && + packet->payload[1] == 0xAD && + packet->payload[2] == 0xBE && + packet->payload[3] == 0xEF)) + { + NDPI_LOG_INFO(ndpi_struct, "found Amazon Video on UDP\n"); + ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_AMAZON_VIDEO, NDPI_PROTOCOL_UNKNOWN); + } else { + NDPI_EXCLUDE_PROTO(ndpi_struct, flow); + } +} + +void ndpi_search_amazon_video(struct ndpi_detection_module_struct *ndpi_struct, struct ndpi_flow_struct *flow) +{ + struct ndpi_packet_struct *packet = &flow->packet; + + NDPI_LOG_DBG(ndpi_struct, "search amazon_video\n"); + + /* skip marked packets */ + if(packet->detected_protocol_stack[0] != NDPI_PROTOCOL_AMAZON_VIDEO) + ndpi_check_amazon_video(ndpi_struct, flow); +} + + +void init_amazon_video_dissector(struct ndpi_detection_module_struct *ndpi_struct, u_int32_t *id, NDPI_PROTOCOL_BITMASK *detection_bitmask) +{ + ndpi_set_bitmask_protocol_detection("AMAZON_VIDEO", ndpi_struct, detection_bitmask, *id, + NDPI_PROTOCOL_AMAZON_VIDEO, + ndpi_search_amazon_video, + NDPI_SELECTION_BITMASK_PROTOCOL_TCP_OR_UDP_WITH_PAYLOAD, + SAVE_DETECTION_BITMASK_AS_UNKNOWN, + ADD_TO_DETECTION_BITMASK); + *id += 1; +} diff --git a/src/lib/protocols/stun.c b/src/lib/protocols/stun.c index b53bed73d..202b3d939 100644 --- a/src/lib/protocols/stun.c +++ b/src/lib/protocols/stun.c @@ -30,6 +30,8 @@ #define MAX_NUM_STUN_PKTS 8 +// #define DEBUG_STUN 1 + struct stun_packet_header { u_int16_t msg_type, msg_len; u_int32_t cookie; @@ -46,6 +48,17 @@ typedef enum { NDPI_IS_NOT_STUN } ndpi_int_stun_t; + +static int is_google_ip_address(u_int32_t host) { + if( + ((host & 0xFFFF0000 /* 255.255.0.0 */) == 0x4A7D0000 /* 74.125.0.0/16 */) + || ((host & 0xFFFF0000 /* 255.255.0.0 */) == 0x42660000 /* 66.102.0.0/16 */) + ) + return(1); + else + return(0); +} + static ndpi_int_stun_t ndpi_int_check_stun(struct ndpi_detection_module_struct *ndpi_struct, struct ndpi_flow_struct *flow, const u_int8_t * payload, @@ -87,10 +100,15 @@ static ndpi_int_stun_t ndpi_int_check_stun(struct ndpi_detection_module_struct * if(msg_type == 0x01 /* Binding Request */) { flow->protos.stun_ssl.stun.num_binding_requests++; - if((msg_len == 0) && (flow->guessed_host_protocol_id == NDPI_PROTOCOL_GOOGLE)) { + if((msg_len == 0) && (flow->guessed_host_protocol_id == NDPI_PROTOCOL_GOOGLE)) { flow->guessed_host_protocol_id = NDPI_PROTOCOL_HANGOUT_DUO; } } + + if((msg_len == 0) && (flow->guessed_host_protocol_id == NDPI_PROTOCOL_UNKNOWN)) { + NDPI_EXCLUDE_PROTO(ndpi_struct, flow); + return(NDPI_IS_NOT_STUN); + } flow->protos.stun_ssl.stun.num_udp_pkts++; @@ -199,9 +217,16 @@ static ndpi_int_stun_t ndpi_int_check_stun(struct ndpi_detection_module_struct * } break; + case 0xFF03: + can_this_be_whatsapp_voice = 0; + flow->guessed_host_protocol_id = NDPI_PROTOCOL_HANGOUT_DUO; + break; + default: /* This means this STUN packet cannot be confused with whatsapp voice */ - /* printf("==> %04X\n", attribute); */ +#ifdef DEBUG_STUN + printf("==> %04X\n", attribute); +#endif can_this_be_whatsapp_voice = 0; break; } @@ -223,8 +248,18 @@ static ndpi_int_stun_t ndpi_int_check_stun(struct ndpi_detection_module_struct * udp_stun_found: if(can_this_be_whatsapp_voice) { + struct ndpi_packet_struct *packet = &flow->packet; + flow->protos.stun_ssl.stun.num_processed_pkts++; - flow->guessed_host_protocol_id = NDPI_PROTOCOL_WHATSAPP_VOICE; +#ifdef DEBUG_STUN + printf("==>> NDPI_PROTOCOL_WHATSAPP_VOICE\n"); +#endif + + if((ntohs(packet->udp->source) == 3478) || (ntohs(packet->udp->dest) == 3478)) + flow->guessed_host_protocol_id = NDPI_PROTOCOL_WHATSAPP_VOICE; + else + flow->guessed_host_protocol_id = (is_google_ip_address(ntohl(packet->iph->saddr)) || is_google_ip_address(ntohl(packet->iph->daddr))) + ? NDPI_PROTOCOL_HANGOUT_DUO : NDPI_PROTOCOL_WHATSAPP_VOICE; return((flow->protos.stun_ssl.stun.num_udp_pkts < MAX_NUM_STUN_PKTS) ? NDPI_IS_NOT_STUN : NDPI_IS_STUN); } else { /* @@ -243,7 +278,12 @@ void ndpi_search_stun(struct ndpi_detection_module_struct *ndpi_struct, struct n NDPI_LOG_DBG(ndpi_struct, "search stun\n"); - if(packet->payload == NULL) return; + if(packet->payload == NULL) + return; + else if(packet->iphv6 != NULL) { + NDPI_EXCLUDE_PROTO(ndpi_struct, flow); + return; + } if(packet->tcp) { /* STUN may be encapsulated in TCP packets */ diff --git a/src/lib/protocols/targus_getdata.c b/src/lib/protocols/targus_getdata.c new file mode 100644 index 000000000..4ee53e8ff --- /dev/null +++ b/src/lib/protocols/targus_getdata.c @@ -0,0 +1,77 @@ +/* + * targus_getdata.c + * + * Copyright (C) 2018 by ntop.org + * + * This file is part of nDPI, an open source deep packet inspection + * library based on the OpenDPI and PACE technology by ipoque GmbH + * + * nDPI is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * nDPI is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with nDPI. If not, see <http://www.gnu.org/licenses/>. + * + */ + +#include "ndpi_protocol_ids.h" + +#define NDPI_CURRENT_PROTO NDPI_PROTOCOL_TARGUS_GETDATA + +#include "ndpi_api.h" + +static void ndpi_check_targus_getdata(struct ndpi_detection_module_struct *ndpi_struct, + struct ndpi_flow_struct *flow) { + struct ndpi_packet_struct *packet = &flow->packet; + + if(packet->iph) { + u_int16_t targus_getdata_port = ntohs(5201); + u_int16_t complex_link_port = ntohs(5001); + + if(((packet->tcp != NULL) && ((packet->tcp->dest == targus_getdata_port) + || (packet->tcp->source == targus_getdata_port) + || (packet->tcp->dest == complex_link_port) + || (packet->tcp->source == complex_link_port))) + || ((packet->udp != NULL) && ((packet->udp->dest == targus_getdata_port) + || (packet->udp->source == targus_getdata_port) + || (packet->udp->dest == complex_link_port) + || (packet->udp->source == complex_link_port)))) { + + NDPI_LOG_INFO(ndpi_struct, "found targus getdata used for speedtest\n"); + ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_TARGUS_GETDATA, NDPI_PROTOCOL_UNKNOWN); + return; + } + } + + NDPI_EXCLUDE_PROTO(ndpi_struct, flow); +} + +void ndpi_search_targus_getdata(struct ndpi_detection_module_struct *ndpi_struct, struct ndpi_flow_struct *flow) +{ + struct ndpi_packet_struct *packet = &flow->packet; + + NDPI_LOG_DBG(ndpi_struct, "search targus getdata\n"); + + /* skip marked packets */ + if(packet->detected_protocol_stack[0] != NDPI_PROTOCOL_TARGUS_GETDATA) + ndpi_check_targus_getdata(ndpi_struct, flow); +} + + +void init_targus_getdata_dissector(struct ndpi_detection_module_struct *ndpi_struct, u_int32_t *id, NDPI_PROTOCOL_BITMASK *detection_bitmask) +{ + ndpi_set_bitmask_protocol_detection("TARGUS_GETDATA", ndpi_struct, detection_bitmask, *id, + NDPI_PROTOCOL_TARGUS_GETDATA, + ndpi_search_targus_getdata, + NDPI_SELECTION_BITMASK_PROTOCOL_TCP_OR_UDP, + SAVE_DETECTION_BITMASK_AS_UNKNOWN, + ADD_TO_DETECTION_BITMASK); + *id += 1; +} diff --git a/src/lib/protocols/xbox.c b/src/lib/protocols/xbox.c index 5d1f64d43..768bb7322 100644 --- a/src/lib/protocols/xbox.c +++ b/src/lib/protocols/xbox.c @@ -80,6 +80,11 @@ void ndpi_search_xbox(struct ndpi_detection_module_struct *ndpi_struct, struct n NDPI_LOG_DBG(ndpi_struct, "maybe xbox\n"); flow->l4.udp.xbox_stage++; return; + } else if ((dport == 3075 || dport == 3076 || dport == 3077 || dport == 3078) || + (sport == 3075 || sport == 3076 || sport == 3077 || sport == 3078)) { + ndpi_int_xbox_add_connection(ndpi_struct, flow); + NDPI_LOG_INFO(ndpi_struct, "found xbox udp port connection detected\n"); + return; } /* exclude here all non matched udp traffic, exclude here tcp only if http has been excluded, because xbox could use http */ @@ -96,7 +101,7 @@ void init_xbox_dissector(struct ndpi_detection_module_struct *ndpi_struct, u_int ndpi_set_bitmask_protocol_detection("Xbox", ndpi_struct, detection_bitmask, *id, NDPI_PROTOCOL_XBOX, ndpi_search_xbox, - NDPI_SELECTION_BITMASK_PROTOCOL_V4_V6_TCP_WITH_PAYLOAD, + NDPI_SELECTION_BITMASK_PROTOCOL_V4_V6_UDP_WITH_PAYLOAD, NO_SAVE_DETECTION_BITMASK_AS_UNKNOWN, ADD_TO_DETECTION_BITMASK); |