From b503900b1456e8bd4b60d1deb0ef7cc3665676f1 Mon Sep 17 00:00:00 2001 From: Toni Uhlig Date: Sun, 4 Apr 2021 21:59:14 +0200 Subject: First step of cleaning up the Makefile.in hell. The goal is to get rid of the Makefile.in's (replace it with Automake Makefile.am's) as they duplicate lot's of text. That decreases readability and is in general a bad design pattern. It seems appropriate to use Automake for an Autoconf based project. Currently achieved: * using libtool to build the core library (+libtool's semantic versioning) * out-of-source builds should work right now * introducing Automake based Makefile in src/ * removed some (maybe) unused GIT ignored files * provide some small python fixes Signed-off-by: Toni Uhlig --- .gitignore | 33 +----- .travis.yml | 20 ++-- Makefile.am | 8 +- autogen.sh | 24 ++-- configure.seed | 51 +++++---- example/Makefile.dpdk.in | 2 +- example/Makefile.in | 72 ++++++++---- fuzz/Makefile.am | 261 +++++++++++++++++++++++++++++++++++++++--- libndpi.pc.in | 2 +- packages/ubuntu/Makefile.in | 6 +- packages/ubuntu/configure | 14 ++- python/Makefile.in | 33 +++--- python/ndpi_example.py | 2 + python/ndpi_typestruct.py | 11 +- src/Makefile.am | 248 +++++++++++++++++++++++++++++++++++++++ src/include/ndpi_utils.h | 1 + src/lib/Makefile.in | 82 ------------- tests/Makefile.am | 4 +- tests/dga/Makefile.in | 28 +++-- tests/do-dga.sh | 65 ----------- tests/do-dga.sh.in | 65 +++++++++++ tests/do-unit.sh | 32 ------ tests/do-unit.sh.in | 20 ++++ tests/do.sh.in | 20 ++-- tests/do_valgrind.sh.in | 11 +- tests/unit/Makefile.in | 25 ++-- utils/generate_automake_am.sh | 164 ++++++++++++++++++++++++++ 27 files changed, 953 insertions(+), 351 deletions(-) create mode 100644 src/Makefile.am delete mode 100644 src/lib/Makefile.in delete mode 100755 tests/do-dga.sh create mode 100755 tests/do-dga.sh.in delete mode 100755 tests/do-unit.sh create mode 100755 tests/do-unit.sh.in create mode 100755 utils/generate_automake_am.sh diff --git a/.gitignore b/.gitignore index ebacea3cd..c2c200803 100644 --- a/.gitignore +++ b/.gitignore @@ -1,56 +1,25 @@ *~ /aclocal.m4 /autom4te.cache/ -.autotools -compile /compile /config.guess -/config.h -/config.h.in -/config.log -/config.status /config.sub /configure /configure.ac /configure.tmp -.cproject /depcomp -.dirstamp -/example/.deps/ -/example/Makefile -/example/ndpiReader -/example/Makefile.dpdk -/example/build *.in /install-sh -*.ko -*.la -/libndpi.pc -.libs -/libtool -*.lo /ltmain.sh /m4/libtool.m4 /m4/lt~obsolete.m4 /m4/ltoptions.m4 /m4/ltsugar.m4 /m4/ltversion.m4 -/Makefile /missing -*.[o] -*.obj -*o.cmd -*o.cmd -.project -.settings -/src/include/Makefile +/python/__pycache__/ /src/include/ndpi_config.h /src/include/ndpi_define.h -/src/lib/.deps/ -/src/lib/Makefile -/src/lib/protocols/.deps/ -/src/lib/third_party/src/.deps/ -stamp-h1 *.swp /test-driver /tests/Makefile diff --git a/.travis.yml b/.travis.yml index f4e0e06b3..a43f32d54 100644 --- a/.travis.yml +++ b/.travis.yml @@ -203,25 +203,25 @@ matrix: before_install: - eval "${MATRIX_EVAL}" -before_script: - - ./autogen.sh +#before_script: # - lcov --directory . --zerocounters script: - - if [ -n "$QA_FUZZ" ]; then ./configure --enable-fuzztargets ; else - ./configure ; fi - - if [ -n "$BUILD_DPKG_PACKAGE" ]; then cd ./packages/ubuntu ; - ./configure && dpkg-buildpackage -b -us -uc ; cd ../.. ; fi - - if [ -n "$DISABLE_GCRYPT" ]; then ./configure --disable-gcrypt ; fi + - if [ -n "$QA_FUZZ" ]; then mkdir build && cd build && ../autogen.sh --enable-fuzztargets ; + elif [ -n "$DISABLE_GCRYPT" ]; then mkdir build && cd build && ../autogen.sh --disable-gcrypt ; + else mkdir build && cd build && ../autogen.sh ; fi - make - make -C example ndpiSimpleIntegration - make dist - -#after_script: - cd tests - ./do.sh - - ./do-unit.sh + - test -x ./do-unit.sh && ./do-unit.sh - ./do-dga.sh + - cd .. + - make distclean + - cd .. + - if [ -n "$BUILD_DPKG_PACKAGE" ]; then cd ./packages/ubuntu ; + ./configure && make ; cd ../.. ; fi #after_success: #- cd ${TRAVIS_BUILD_DIR} diff --git a/Makefile.am b/Makefile.am index 789a52ef3..f48b9aa7d 100644 --- a/Makefile.am +++ b/Makefile.am @@ -1,5 +1,5 @@ ACLOCAL_AMFLAGS = -I m4 -SUBDIRS = src/lib @EXTRA_TARGETS@ +SUBDIRS = src @EXTRA_TARGETS@ if BUILD_FUZZTARGETS SUBDIRS += fuzz @@ -8,7 +8,11 @@ endif pkgconfigdir = $(libdir)/pkgconfig pkgconfig_DATA = libndpi.pc -EXTRA_DIST = README.md CHANGELOG.md CONTRIBUTING.md README.protocols autogen.sh configure.seed wireshark python windows utils packages doc/nDPI_QuickStartGuide.pages doc/nDPI_QuickStartGuide.pdf example/MacOS example/Win32 +EXTRA_DIST = \ + README.md CHANGELOG.md CONTRIBUTING.md README.protocols \ + autogen.sh configure.seed \ + wireshark python windows utils packages example/MacOS example/Win32 \ + doc/nDPI_QuickStartGuide.pages doc/nDPI_QuickStartGuide.pdf changelog: git log --since={`curl -s https://github.com/ntop/ndpi/releases | grep datetime | head -n1 | egrep -o "[0-9]+\-[0-9]+\-[0-9]+"`} --name-only --pretty=format:" - %s" | grep "^ " > CHANGELOG.latest diff --git a/autogen.sh b/autogen.sh index 94e1cfd69..6f6a654ca 100755 --- a/autogen.sh +++ b/autogen.sh @@ -5,6 +5,8 @@ NDPI_MINOR="1" NDPI_PATCH="0" NDPI_VERSION_SHORT="$NDPI_MAJOR.$NDPI_MINOR.$NDPI_PATCH" +SCRIPT_DIR="$(dirname ${0})" + rm -f configure config.h config.h.in AUTOCONF=$(command -v autoconf) @@ -45,19 +47,27 @@ if test -z $PKG_CONFIG; then exit fi -cat configure.seed | sed \ +"${SCRIPT_DIR}/utils/generate_automake_am.sh" + +cat "${SCRIPT_DIR}/configure.seed" | sed \ -e "s/@NDPI_MAJOR@/$NDPI_MAJOR/g" \ -e "s/@NDPI_MINOR@/$NDPI_MINOR/g" \ -e "s/@NDPI_PATCH@/$NDPI_PATCH/g" \ -e "s/@NDPI_VERSION_SHORT@/$NDPI_VERSION_SHORT/g" \ -e "s/@FUZZY@/$FUZZY/g" \ - > configure.ac + > "${SCRIPT_DIR}/configure.ac" +OLDPWD="$(pwd)" +cd "${SCRIPT_DIR}" || exit 1 autoreconf -ivf -cat configure | sed "s/#define PACKAGE/#define NDPI_PACKAGE/g" | sed "s/#define VERSION/#define NDPI_VERSION/g" > configure.tmp -cat configure.tmp > configure +cd "${OLDPWD}" || exit 1 -echo "./configure $@" -chmod +x configure -./configure $@ +cat "${SCRIPT_DIR}/configure" | \ + sed "s/#define PACKAGE/#define NDPI_PACKAGE/g" | \ + sed "s/#define VERSION/#define NDPI_VERSION/g" \ + > "${SCRIPT_DIR}/configure.tmp" +cat "${SCRIPT_DIR}/configure.tmp" > "${SCRIPT_DIR}/configure" +echo "${SCRIPT_DIR}/configure ${*}" +chmod +x "${SCRIPT_DIR}/configure" +${SCRIPT_DIR}/configure ${@} diff --git a/configure.seed b/configure.seed index be8a2708c..d08cc9059 100644 --- a/configure.seed +++ b/configure.seed @@ -29,10 +29,19 @@ AM_CONDITIONAL([BUILD_FUZZTARGETS], [test "x$enable_fuzztargets" = "xyes"]) AS_IF([test "${with_sanitizer+set}" = set],[ CFLAGS="${CFLAGS} -g3 -O0 -Wno-unused-value -fsanitize=address -fsanitize=undefined -fno-sanitize=alignment -fsanitize=leak -fno-omit-frame-pointer" LDFLAGS="${LDFLAGS} -fsanitize=address -fsanitize=undefined -fno-sanitize=alignment -fsanitize=leak" -]) + ENABLE_SANITIZER=1 +], [ENABLE_SANITIZER=0]) +AC_ENABLE_SHARED +AC_ENABLE_STATIC LT_INIT LT_LIB_M +LIBS="$LIBS$LIBM " + +if test $srcdir != . && test -d $srcdir/src/include/ndpi_config.h; then + AC_MSG_ERROR([building out of tree but $srcdir contains host-${host_noncanonical}. +Use a pristine source tree when building in a separate tree]) +fi SYSTEM=`uname -s` if test $SYSTEM = "Darwin"; then @@ -48,6 +57,13 @@ dnl> AC_PROG_CC fi AC_LANG_WERROR AX_PTHREAD +AC_TYPE_UINT8_T +AC_TYPE_UINT32_T +AC_TYPE_UINT64_T +AC_TYPE_SIZE_T +AC_CHECK_TYPE(int) +AC_CHECK_TYPE(long) +AC_CHECK_TYPE(time_t) NDPI_VERSION_SHORT="@NDPI_VERSION_SHORT@" NDPI_MAJOR="@NDPI_MAJOR@" @@ -76,7 +92,7 @@ if test -d ".git" || test -f ".git" ; then : NDPI_API_VERSION=$((A+B+C)) else GIT_RELEASE="${PACKAGE_VERSION}" - GIT_DATE=`date -u -r CHANGELOG.md` + GIT_DATE=`date -u -r "${srcdir}/CHANGELOG.md"` if test -z "$SOURCE_DATE_EPOCH" ; then : SOURCE_DATE_EPOCH=`date +%s` fi @@ -93,12 +109,8 @@ if ! test "${with_mipsel+set}" = set && ! test "${with_only_libndpi+set}" = set; PKG_CHECK_MODULES([JSONC], [json-c], [JSONC_LIBS=`pkg-config --libs json-c` JSONC_CFLAGS=`pkg-config --cflags json-c`]) fi -LIBS="$LIBS $JSONC_LIBS" -CFLAGS="$CFLAGS $JSONC_CFLAGS" - AC_CHECK_HEADERS([netinet/in.h stdint.h stdlib.h string.h unistd.h json.h]) -ADDITIONAL_LIBS="$LIBM" PCAP_HOME=$HOME/PF_RING/userland DPDK_TARGET= @@ -205,41 +217,40 @@ AC_ARG_ENABLE([gcrypt], [GCRYPT_ENABLED=0], [AC_CHECK_LIB(gcrypt, gcry_cipher_checktag)]) AS_IF([test "x$enable_gcrypt" = "xyes"], [ - if test "x$ac_cv_lib_gcrypt_gcry_cipher_checktag" = xyes; then : - ADDITIONAL_LIBS="${ADDITIONAL_LIBS} -lgcrypt" - else + if test "x$ac_cv_lib_gcrypt_gcry_cipher_checktag" != xyes; then : $as_unset ac_cv_lib_gcrypt_gcry_cipher_checktag AC_CHECK_LIB(gpg-error, gpg_strerror_r) AC_CHECK_LIB(gcrypt, gcry_cipher_checktag) - if test "x$ac_cv_lib_gcrypt_gcry_cipher_checktag" = xyes -a "x$ac_cv_lib_gpg_error_gpg_strerror_r" = xyes; then : - ADDITIONAL_LIBS="${ADDITIONAL_LIBS} -lgcrypt -lgpg-error" - fi fi ]) dnl> PCRE -AC_ARG_WITH(pcre, [ --with-pcre Enable nDPI build with libpcre]) +AC_ARG_WITH(pcre, AS_HELP_STRING([--with-pcre], [Enable nDPI build with libpcre])) if test "${with_pcre+set}" = set; then : - AC_CHECK_LIB(pcre, pcre_compile, AC_DEFINE_UNQUOTED(HAVE_PCRE, 1, [libpcre(-dev) is present])) + AC_CHECK_LIB([pcre], [pcre_compile]) if test "x$ac_cv_lib_pcre_pcre_compile" = xyes; then : - ADDITIONAL_LIBS="${ADDITIONAL_LIBS} -lpcre" + AC_DEFINE_UNQUOTED(HAVE_PCRE, 1, [libpcre(-dev) is present]) fi fi dnl> GeoIP -AC_ARG_WITH(maxminddb, [ --with-maxminddb Enable nDPI build with libmaxminddb]) +AC_ARG_WITH(maxminddb, AS_HELP_STRING([--with-maxminddb], [Enable nDPI build with libmaxminddb])) if test "${with_maxminddb+set}" = set; then : AC_CHECK_LIB([maxminddb], [MMDB_lookup_sockaddr]) AC_HAVE_HEADERS(maxminddb.h) if test ".${ac_cv_lib_maxminddb_MMDB_lookup_sockaddr}" = ".yes" && test ".${ac_cv_header_maxminddb_h}" = ".yes"; then - ADDITIONAL_LIBS="${ADDITIONAL_LIBS} -lmaxminddb" AC_DEFINE_UNQUOTED(HAVE_MAXMINDDB, 1, [MaxMind DB support]) fi fi -AC_CONFIG_FILES([Makefile example/Makefile example/Makefile.dpdk tests/Makefile tests/unit/Makefile tests/dga/Makefile libndpi.pc src/include/ndpi_define.h src/lib/Makefile python/Makefile fuzz/Makefile src/include/ndpi_api.h]) +AC_CONFIG_FILES([Makefile example/Makefile example/Makefile.dpdk tests/Makefile tests/unit/Makefile tests/dga/Makefile libndpi.pc src/include/ndpi_define.h src/Makefile python/Makefile fuzz/Makefile src/include/ndpi_api.h]) AC_CONFIG_FILES([tests/do.sh], [chmod +x tests/do.sh]) -AC_CONFIG_FILES([tests/do_valgrind.sh], [chmod +x tests/do_valgrind.sh]) +AS_IF([test "${with_sanitizer+set}" = set],[],[ + AC_CONFIG_FILES([tests/do_valgrind.sh], [chmod +x tests/do_valgrind.sh]) +dnl> Skipping tests with sanitizer enabled due to use-of-uninitialized-value in json-c + AC_CONFIG_FILES([tests/do-unit.sh], [chmod +x tests/do-unit.sh]) +]) +AC_CONFIG_FILES([tests/do-dga.sh], [chmod +x tests/do-dga.sh]) AC_CONFIG_HEADERS(src/include/ndpi_config.h) AC_SUBST(GIT_RELEASE) AC_SUBST(NDPI_MAJOR) @@ -248,7 +259,6 @@ AC_SUBST(NDPI_PATCH) AC_SUBST(NDPI_VERSION_SHORT) AC_SUBST(PCAP_INC) AC_SUBST(PCAP_LIB) -AC_SUBST(ADDITIONAL_LIBS) AC_SUBST(DL_LIB) AC_SUBST(DPDK_TARGET) AC_SUBST(HAVE_PTHREAD_SETAFFINITY_NP) @@ -257,6 +267,7 @@ AC_SUBST(NDPI_API_VERSION) AC_SUBST(EXTRA_TARGETS) AC_SUBST(BUILD_MINGW) AC_SUBST(BUILD_FUZZTARGETS) +AC_SUBST(ENABLE_SANITIZER) AC_SUBST(JSONC_CFLAGS) AC_SUBST(JSONC_LIBS) AC_SUBST(GCRYPT_ENABLED) diff --git a/example/Makefile.dpdk.in b/example/Makefile.dpdk.in index cd703414b..016a6520c 100644 --- a/example/Makefile.dpdk.in +++ b/example/Makefile.dpdk.in @@ -15,7 +15,7 @@ RTE_TARGET ?= x86_64-native-linuxapp-gcc include $(RTE_SDK)/mk/rte.vars.mk APP = ndpiReader.dpdk -LIBNDPI = $(PWD)/../src/lib/libndpi.a +LIBNDPI = ../src/.libs/libndpi.a SRCS-y := reader_util.c intrusion_detection.c ndpiReader.c diff --git a/example/Makefile.in b/example/Makefile.in index cef1edb63..79847087a 100644 --- a/example/Makefile.in +++ b/example/Makefile.in @@ -1,12 +1,20 @@ CC=@CC@ CXX=@CXX@ BUILD_MINGW=@BUILD_MINGW@ -SRCHOME=../src -CFLAGS=-g -fPIC -DPIC -I$(SRCHOME)/include @PCAP_INC@ @CFLAGS@ -LIBNDPI=$(SRCHOME)/lib/libndpi.a -LDFLAGS=$(LIBNDPI) @PCAP_LIB@ @LIBS@ @ADDITIONAL_LIBS@ -lpthread @LDFLAGS@ -HEADERS=intrusion_detection.h reader_util.h $(SRCHOME)/include/ndpi_api.h \ - $(SRCHOME)/include/ndpi_typedefs.h $(SRCHOME)/include/ndpi_protocol_ids.h +DPDK_TARGET=@DPDK_TARGET@ + +BUILD_SRC=../src +ROOT_SRC=@srcdir@/../src +ROOT_EXAMPLES=@srcdir@ + +CFLAGS=-g -fPIC -DPIC -I$(ROOT_SRC)/include -I$(BUILD_SRC)/include @PCAP_INC@ @CFLAGS@ +LIBNDPI=$(BUILD_SRC)/.libs/libndpi.a +LDFLAGS=$(LIBNDPI) @PCAP_LIB@ @LIBS@ -lpthread @LDFLAGS@ +HEADERS=$(ROOT_EXAMPLES)/intrusion_detection.h \ + $(ROOT_EXAMPLES)/reader_util.h \ + $(BUILD_SRC)/include/ndpi_api.h \ + $(ROOT_SRC)/include/ndpi_typedefs.h \ + $(ROOT_SRC)/include/ndpi_protocol_ids.h OBJS=ndpiReader.o reader_util.o intrusion_detection.o PREFIX?=@prefix@ @@ -16,47 +24,67 @@ all: else -all: ndpiReader @DPDK_TARGET@ - -EXECUTABLE_SOURCES := ndpiReader.c ndpiSimpleIntegration.c -COMMON_SOURCES := $(filter-out $(EXECUTABLE_SOURCES),$(wildcard *.c )) +all: ndpiReader $(DPDK_TARGET) -libndpiReader.a: $(COMMON_SOURCES:%.c=%.o) $(LIBNDPI) - ar rsv libndpiReader.a $(COMMON_SOURCES:%.c=%.o) +libndpiReader.a: $(LIBNDPI) reader_util.o intrusion_detection.o + ar rsv libndpiReader.a reader_util.o intrusion_detection.o -ndpiReader: libndpiReader.a $(LIBNDPI) ndpiReader.o - $(CC) $(CFLAGS) ndpiReader.o libndpiReader.a -o $@ $(LDFLAGS) +ndpiReader: libndpiReader.a ndpiReader.o + $(CC) $(CFLAGS) ndpiReader.o -o $@ libndpiReader.a $(LDFLAGS) ndpiSimpleIntegration: ndpiSimpleIntegration.o $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) -%.o: %.c $(HEADERS) Makefile - $(CC) $(CFLAGS) -c $< -o $@ +%.o: $(ROOT_EXAMPLES)/%.c $(HEADERS) Makefile + $(CC) $(CFLAGS) -c $< -o $(notdir $@) install: ndpiReader mkdir -p $(DESTDIR)$(PREFIX)/bin/ mkdir -p $(DESTDIR)$(PREFIX)/share/ndpi cp ndpiReader $(DESTDIR)$(PREFIX)/bin/ - cp protos.txt $(DESTDIR)$(PREFIX)/share/ndpi/ndpiProtos.txt - cp mining_hosts.txt $(DESTDIR)$(PREFIX)/share/ndpi/ndpiCustomCategory.txt + cp $(ROOT_EXAMPLES)/protos.txt $(DESTDIR)$(PREFIX)/share/ndpi/ndpiProtos.txt + cp $(ROOT_EXAMPLES)/categories.txt $(DESTDIR)$(PREFIX)/share/ndpi/ndpiCustomCategory.txt + cp $(ROOT_EXAMPLES)/mining_hosts.txt $(DESTDIR)$(PREFIX)/share/ndpi/ndpiCustomHosts.txt [ -f build/app/ndpiReader.dpdk ] && cp build/app/ndpiReader.dpdk $(DESTDIR)$(PREFIX)/bin/ || true [ -f ndpiReader.dpdk ] && cp ndpiReader.dpdk $(DESTDIR)$(PREFIX)/bin/ || true dpdk: +ifneq ($(DPDK_TARGET),) make -f Makefile.dpdk +else + @echo 'Skipping. DPDK needs to be enabled during configure.' +endif check: - cppcheck --template='{file}:{line}:{severity}:{message}' --quiet --enable=all --force -I$(SRCHOME)/include *.c + cppcheck --template='{file}:{line}:{severity}:{message}' --quiet --enable=all --force \ + -I$(ROOT_SRC)/include \ + -I$(BUILD_SRC)/include \ + -I. \ + $(ROOT_EXAMPLES)/*.c clean: - /bin/rm -f *.o ndpiReader ndpiReader.dpdk + /bin/rm -f *.o ndpiReader ndpiReader.dpdk ndpiSimpleIntegration libndpiReader.a /bin/rm -f .*.dpdk.cmd .*.o.cmd *.dpdk.map .*.o.d /bin/rm -f _install _postbuild _postinstall _preinstall /bin/rm -rf build distdir: - cp categories.txt mining_hosts.txt protos.txt README.DPDK '$(distdir)/' - find . -maxdepth 1 -type f -name '*.c' -o -name '*.h' -o -name '*.py' | xargs -I'{}' cp '{}' '$(distdir)/{}' + cp \ + $(ROOT_EXAMPLES)/categories.txt \ + $(ROOT_EXAMPLES)/mining_hosts.txt \ + $(ROOT_EXAMPLES)/protos.txt \ + $(ROOT_EXAMPLES)/risky_domains.txt \ + $(ROOT_EXAMPLES)/ja3_fingerprints.csv \ + $(ROOT_EXAMPLES)/sha1_fingerprints.csv \ + $(ROOT_EXAMPLES)/README.DPDK \ + $(ROOT_EXAMPLES)/intrusion_detection.h \ + $(ROOT_EXAMPLES)/ndpiSimpleIntegration.c \ + $(ROOT_EXAMPLES)/reader_util.c \ + $(ROOT_EXAMPLES)/ndpiReader.c \ + $(ROOT_EXAMPLES)/ndpi2timeline.py \ + $(ROOT_EXAMPLES)/intrusion_detection.c \ + $(ROOT_EXAMPLES)/reader_util.h \ + '$(distdir)/' distclean: clean /bin/rm -f Makefile.dpdk diff --git a/fuzz/Makefile.am b/fuzz/Makefile.am index 7858c3371..3e8ff277b 100644 --- a/fuzz/Makefile.am +++ b/fuzz/Makefile.am @@ -1,43 +1,268 @@ bin_PROGRAMS = fuzz_process_packet fuzz_ndpi_reader fuzz_ndpi_reader_with_main +BUILD_SRC=../src +BUILD_EXAMPLE=../example + fuzz_process_packet_SOURCES = fuzz_process_packet.c -fuzz_process_packet_CFLAGS = -fuzz_process_packet_LDADD = ../src/lib/libndpi.a -fuzz_process_packet_LDFLAGS = $(ADDITIONAL_LIBS) $(LIBS) +fuzz_process_packet_CFLAGS = -I$(top_srcdir)/example -I$(top_srcdir)/src/include -I$(BUILD_SRC)/include +fuzz_process_packet_LDADD = $(BUILD_SRC)/.libs/libndpi.a +fuzz_process_packet_LDFLAGS = $(LIBS) if HAS_FUZZLDFLAGS fuzz_process_packet_CFLAGS += $(LIB_FUZZING_ENGINE) fuzz_process_packet_LDFLAGS += $(LIB_FUZZING_ENGINE) endif # force usage of CXX for linker fuzz_process_packet_LINK=$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \ - $(LIBTOOLFLAGS) --mode=link $(CXX) $(AM_CXXFLAGS) $(CXXFLAGS) \ - $(fuzz_process_packet_LDFLAGS) $(LDFLAGS) -o $@ + $(LIBTOOLFLAGS) --mode=link $(CXX) $(AM_CXXFLAGS) $(CXXFLAGS) \ + $(fuzz_process_packet_LDFLAGS) $(LDFLAGS) -o $@ fuzz_ndpi_reader_SOURCES = fuzz_ndpi_reader.c -fuzz_ndpi_reader_CFLAGS = -I../example/ -fuzz_ndpi_reader_LDADD = ../example/libndpiReader.a ../src/lib/libndpi.a -fuzz_ndpi_reader_LDFLAGS = $(PCAP_LIB) $(ADDITIONAL_LIBS) $(LIBS) +fuzz_ndpi_reader_CFLAGS = -I$(top_srcdir)/example -I$(top_srcdir)/src/include -I$(BUILD_SRC)/include +fuzz_ndpi_reader_LDADD = $(BUILD_EXAMPLE)/libndpiReader.a $(BUILD_SRC)/.libs/libndpi.a +fuzz_ndpi_reader_LDFLAGS = $(PCAP_LIB) $(LIBS) if HAS_FUZZLDFLAGS fuzz_ndpi_reader_CFLAGS += $(LIB_FUZZING_ENGINE) fuzz_ndpi_reader_LDFLAGS += $(LIB_FUZZING_ENGINE) endif # force usage of CXX for linker fuzz_ndpi_reader_LINK=$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \ - $(LIBTOOLFLAGS) --mode=link $(CXX) $(AM_CXXFLAGS) $(CXXFLAGS) \ - $(fuzz_ndpi_reader_LDFLAGS) $(LDFLAGS) -o $@ + $(LIBTOOLFLAGS) --mode=link $(CXX) $(AM_CXXFLAGS) $(CXXFLAGS) \ + $(fuzz_ndpi_reader_LDFLAGS) $(LDFLAGS) -o $@ fuzz_ndpi_reader_with_main_SOURCES = fuzz_ndpi_reader.c -fuzz_ndpi_reader_with_main_CFLAGS = -I../example/ -DBUILD_MAIN -fuzz_ndpi_reader_with_main_LDADD = ../src/lib/libndpi.a -fuzz_ndpi_reader_with_main_LDFLAGS = ../example/libndpiReader.a $(PCAP_LIB) $(ADDITIONAL_LIBS) $(LIBS) +fuzz_ndpi_reader_with_main_CFLAGS = -I$(top_srcdir)/example -I$(top_srcdir)/src/include -I$(BUILD_SRC)/include -DBUILD_MAIN +fuzz_ndpi_reader_with_main_LDADD = $(BUILD_SRC)/.libs/libndpi.a +fuzz_ndpi_reader_with_main_LDFLAGS = $(BUILD_EXAMPLE)/libndpiReader.a $(PCAP_LIB) $(LIBS) # force usage of CXX for linker fuzz_ndpi_reader_with_main_LINK=$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \ - $(LIBTOOLFLAGS) --mode=link $(CXX) $(AM_CXXFLAGS) $(CXXFLAGS) \ - $(fuzz_ndpi_reader_with_main_LDFLAGS) $(LDFLAGS) -o $@ + $(LIBTOOLFLAGS) --mode=link $(CXX) $(AM_CXXFLAGS) $(CXXFLAGS) \ + $(fuzz_ndpi_reader_with_main_LDFLAGS) $(LDFLAGS) -o $@ # required for Google oss-fuzz # see https://github.com/google/oss-fuzz/tree/master/projects/ndpi -testpcaps := $(wildcard ../tests/pcap/*.pcap) +TESTPCAPS = \ + ./1kxun.pcap \ + ./443-chrome.pcap \ + ./443-curl.pcap \ + ./443-firefox.pcap \ + ./443-git.pcap \ + ./443-opvn.pcap \ + ./443-safari.pcap \ + ./4in4tunnel.pcap \ + ./4in6tunnel.pcap \ + ./6in4tunnel.pcap \ + ./6in6tunnel.pcap \ + ./aimini-http.pcap \ + ./ajp.pcap \ + ./alexa-app.pcapng \ + ./among_us.pcap \ + ./amqp.pcap \ + ./android.pcap \ + ./anyconnect-vpn.pcap \ + ./anydesk-2.pcap \ + ./anydesk.pcap \ + ./avast_securedns.pcapng \ + ./bad-dns-traffic.pcap \ + ./badpackets.pcap \ + ./BGP_Cisco_hdlc_slarp.pcap \ + ./BGP_redist.pcap \ + ./bitcoin.pcap \ + ./bittorrent_ip.pcap \ + ./bittorrent.pcap \ + ./bittorrent_utp.pcap \ + ./bt_search.pcap \ + ./capwap.pcap \ + ./check_mk_new.pcap \ + ./chrome.pcap \ + ./coap_mqtt.pcap \ + ./cpha.pcap \ + ./dcerpc.pcap \ + ./diameter.pcap \ + ./dlt_ppp.pcap \ + ./dnp3.pcap \ + ./dns_ambiguous_names.pcap \ + ./dnscrypt_skype_false_positive.pcapng \ + ./dnscrypt-v1-and-resolver-pings.pcap \ + ./dnscrypt-v2-doh.pcap \ + ./dns_doh.pcap \ + ./dns_dot.pcap \ + ./dns_exfiltration.pcap \ + ./dns_long_domainname.pcap \ + ./dns-tunnel-iodine.pcap \ + ./doq_adguard.pcapng \ + ./doq.pcapng \ + ./dos_win98_smb_netbeui.pcap \ + ./drda_db2.pcap \ + ./dropbox.pcap \ + ./dtls2.pcap \ + ./dtls_certificate_fragments.pcap \ + ./dtls.pcap \ + ./dtls_session_id_and_coockie_both.pcap \ + ./EAQ.pcap \ + ./encrypted_sni.pcap \ + ./ethereum.pcap \ + ./exe_download_as_png.pcap \ + ./exe_download.pcap \ + ./facebook.pcap \ + ./firefox.pcap \ + ./fix.pcap \ + ./forticlient.pcap \ + ./ftp_failed.pcap \ + ./ftp.pcap \ + ./fuzz-2006-06-26-2594.pcap \ + ./fuzz-2006-09-29-28586.pcap \ + ./fuzz-2020-02-16-11740.pcap \ + ./fuzz-2021-06-07-c6c72a0a56.pcap \ + ./genshin-impact.pcap \ + ./git.pcap \ + ./googledns_android10.pcap \ + ./google_ssl.pcap \ + ./gquic.pcap \ + ./h323-overflow.pcap \ + ./hangout.pcap \ + ./hpvirtgrp.pcap \ + ./http-crash-content-disposition.pcap \ + ./http_ipv6.pcap \ + ./http-lines-split.pcap \ + ./IEC104.pcap \ + ./iec60780-5-104.pcap \ + ./imaps.pcap \ + ./instagram.pcap \ + ./ip_fragmented_garbage.pcap \ + ./iphone.pcap \ + ./ipv6_in_gtp.pcap \ + ./irc.pcap \ + ./ja3_lots_of_cipher_suites_2_anon.pcap \ + ./ja3_lots_of_cipher_suites.pcap \ + ./KakaoTalk_chat.pcap \ + ./KakaoTalk_talk.pcap \ + ./kerberos.pcap \ + ./long_tls_certificate.pcap \ + ./malformed_dns.pcap \ + ./malformed_icmp.pcap \ + ./malware.pcap \ + ./modbus.pcap \ + ./monero.pcap \ + ./mongodb.pcap \ + ./mpeg.pcap \ + ./mpegts.pcap \ + ./mssql_tds.pcap \ + ./mysql-8.pcap \ + ./nats.pcap \ + ./ndpi_match_string_subprotocol__error.pcapng \ + ./nest_log_sink.pcap \ + ./netbios.pcap \ + ./netbios_wildcard_dns_query.pcap \ + ./netflix.pcap \ + ./netflow-fritz.pcap \ + ./netflowv9.pcap \ + ./nintendo.pcap \ + ./no_sni.pcap \ + ./NTPv2.pcap \ + ./NTPv3.pcap \ + ./NTPv4.pcap \ + ./ocs.pcap \ + ./ookla.pcap \ + ./openvpn.pcap \ + ./Oscar.pcap \ + ./os_detected.pcapng \ + ./pinterest.pcap \ + ./pps.pcap \ + ./ps_vue.pcap \ + ./quic046.pcap \ + ./quic_0RTT.pcap \ + ./quic-23.pcap \ + ./quic-24.pcap \ + ./quic-27.pcap \ + ./quic-28.pcap \ + ./quic-29.pcap \ + ./quic-33.pcapng \ + ./quic_frags_ch_in_multiple_packets.pcapng \ + ./quic_frags_ch_out_of_order_same_packet_craziness.pcapng \ + ./quic_interop_V.pcapng \ + ./quickplay.pcap \ + ./quic-mvfst-22_decryption_error.pcap \ + ./quic-mvfst-22.pcap \ + ./quic-mvfst-27.pcapng \ + ./quic-mvfst-exp.pcap \ + ./quic.pcap \ + ./quic_q39.pcap \ + ./quic_q43.pcap \ + ./quic_q46_b.pcap \ + ./quic_q46.pcap \ + ./quic_q50.pcap \ + ./quic_t50.pcap \ + ./quic_t51.pcap \ + ./rdp.pcap \ + ./reasm_crash_anon.pcapng \ + ./reasm_segv_anon.pcapng \ + ./reddit.pcap \ + ./rtsp_setup_http.pcapng \ + ./rx.pcap \ + ./s7comm.pcap \ + ./safari.pcap \ + ./selfsigned.pcap \ + ./signal.pcap \ + ./simple-dnscrypt.pcap \ + ./sip.pcap \ + ./skype-conference-call.pcap \ + ./skype_no_unknown.pcap \ + ./skype.pcap \ + ./skype_udp.pcap \ + ./smb_deletefile.pcap \ + ./smbv1.pcap \ + ./smpp_in_general.pcap \ + ./snapchat_call.pcapng \ + ./snapchat.pcap \ + ./ssdp-m-search.pcap \ + ./ssh.pcap \ + ./ssl-cert-name-mismatch.pcap \ + ./starcraft_battle.pcap \ + ./steam_datagram_relay_ping.pcapng \ + ./steam.pcap \ + ./synscan.pcap \ + ./teams.pcap \ + ./teamspeak3.pcap \ + ./telegram.pcap \ + ./teredo.pcap \ + ./tftp.pcap \ + ./tinc.pcap \ + ./tk.pcap \ + ./tls_certificate_too_long.pcap \ + ./tls-esni-fuzzed.pcap \ + ./tls_esni_sni_both.pcap \ + ./tls_invalid_reads.pcap \ + ./tls_long_cert.pcap \ + ./tls-rdn-extract.pcap \ + ./tls_verylong_certificate.pcap \ + ./tor.pcap \ + ./trickbot.pcap \ + ./tumblr.pcap \ + ./ubntac2.pcap \ + ./upnp.pcap \ + ./viber.pcap \ + ./vnc.pcap \ + ./wa_video.pcap \ + ./wa_voice.pcap \ + ./waze.pcap \ + ./WebattackRCE.pcap \ + ./WebattackSQLinj.pcap \ + ./WebattackXSS.pcap \ + ./webex.pcap \ + ./websocket.pcap \ + ./wechat.pcap \ + ./weibo.pcap \ + ./whatsappfiles.pcap \ + ./whatsapp_login_call.pcap \ + ./whatsapp_login_chat.pcap \ + ./whatsapp_voice_and_message.pcap \ + ./wireguard.pcap \ + ./youtube_quic.pcap \ + ./youtubeupload.pcap \ + ./z3950.pcapng \ + ./zabbix.pcap \ + ./zcash.pcap \ + ./zoom.pcap -fuzz_ndpi_reader_seed_corpus.zip: $(testpcaps) - zip -r fuzz_ndpi_reader_seed_corpus.zip $(testpcaps) +fuzz_ndpi_reader_seed_corpus.zip: $(TESTPCAPS) + zip -r fuzz_ndpi_reader_seed_corpus.zip $(TESTPCAPS) diff --git a/libndpi.pc.in b/libndpi.pc.in index b67b2d700..02dfd6cb3 100644 --- a/libndpi.pc.in +++ b/libndpi.pc.in @@ -6,5 +6,5 @@ includedir=@includedir@ Name: libndpi Description: deep packet inspection library Version: @VERSION@ -Libs: -L${libdir} -lndpi @ADDITIONAL_LIBS@ +Libs: -L${libdir} -lndpi @LIBS@ Cflags: -I${includedir}/ndpi diff --git a/packages/ubuntu/Makefile.in b/packages/ubuntu/Makefile.in index f806352cb..fe6b0ee81 100644 --- a/packages/ubuntu/Makefile.in +++ b/packages/ubuntu/Makefile.in @@ -7,10 +7,8 @@ ndpi: mkdir -p ./debian/ndpi-tmp/usr/lib ./debian/ndpi-tmp/usr/bin mkdir -p ./debian/ndpi-dev-tmp/usr/lib ./debian/ndpi-dev-tmp/usr/include/ndpi cd ${NDPI_HOME}; ./autogen.sh; ./configure; make - cp $(NDPI_HOME)/src/lib/libndpi.so.@NDPI_VERS@ ./debian/ndpi-tmp/usr/lib/ - cd ./debian/ndpi-tmp/usr/lib/; ln -s libndpi.so.@NDPI_VERS@ libndpi.so; cd - - cd ./debian/ndpi-tmp/usr/lib/; ln -s libndpi.so.@NDPI_VERS@ libndpi.so.@MAJOR_RELEASE@; cd - - cp $(NDPI_HOME)/src/lib/libndpi.a ./debian/ndpi-dev-tmp/usr/lib/ + cp $(NDPI_HOME)/src/.libs/libndpi.so* ./debian/ndpi-tmp/usr/lib/ + cp $(NDPI_HOME)/src/.libs/libndpi.a ./debian/ndpi-dev-tmp/usr/lib/ cp $(NDPI_HOME)/example/ndpiReader ./debian/ndpi-tmp/usr/bin/ cp $(NDPI_HOME)/src/include/*.h ./debian/ndpi-dev-tmp/usr/include/ndpi/ -rm -fr ./debian/ndpi-dev-tmp/usr/include/ndpi/ndpi_win32.h* diff --git a/packages/ubuntu/configure b/packages/ubuntu/configure index ae5f07ec0..08a06ef0a 100755 --- a/packages/ubuntu/configure +++ b/packages/ubuntu/configure @@ -657,6 +657,7 @@ infodir docdir oldincludedir includedir +runstatedir localstatedir sharedstatedir sysconfdir @@ -727,6 +728,7 @@ datadir='${datarootdir}' sysconfdir='${prefix}/etc' sharedstatedir='${prefix}/com' localstatedir='${prefix}/var' +runstatedir='${localstatedir}/run' includedir='${prefix}/include' oldincludedir='/usr/include' docdir='${datarootdir}/doc/${PACKAGE_TARNAME}' @@ -979,6 +981,15 @@ do | -silent | --silent | --silen | --sile | --sil) silent=yes ;; + -runstatedir | --runstatedir | --runstatedi | --runstated \ + | --runstate | --runstat | --runsta | --runst | --runs \ + | --run | --ru | --r) + ac_prev=runstatedir ;; + -runstatedir=* | --runstatedir=* | --runstatedi=* | --runstated=* \ + | --runstate=* | --runstat=* | --runsta=* | --runst=* | --runs=* \ + | --run=* | --ru=* | --r=*) + runstatedir=$ac_optarg ;; + -sbindir | --sbindir | --sbindi | --sbind | --sbin | --sbi | --sb) ac_prev=sbindir ;; -sbindir=* | --sbindir=* | --sbindi=* | --sbind=* | --sbin=* \ @@ -1116,7 +1127,7 @@ fi for ac_var in exec_prefix prefix bindir sbindir libexecdir datarootdir \ datadir sysconfdir sharedstatedir localstatedir includedir \ oldincludedir docdir infodir htmldir dvidir pdfdir psdir \ - libdir localedir mandir + libdir localedir mandir runstatedir do eval ac_val=\$$ac_var # Remove trailing slashes. @@ -1269,6 +1280,7 @@ Fine tuning of the installation directories: --sysconfdir=DIR read-only single-machine data [PREFIX/etc] --sharedstatedir=DIR modifiable architecture-independent data [PREFIX/com] --localstatedir=DIR modifiable single-machine data [PREFIX/var] + --runstatedir=DIR modifiable per-process data [LOCALSTATEDIR/run] --libdir=DIR object code libraries [EPREFIX/lib] --includedir=DIR C header files [PREFIX/include] --oldincludedir=DIR C header files for non-gcc [/usr/include] diff --git a/python/Makefile.in b/python/Makefile.in index a3bcd5da3..1a5929bec 100644 --- a/python/Makefile.in +++ b/python/Makefile.in @@ -1,26 +1,33 @@ CC=@CC@ -CFLAGS=-I. -I../src/include -I./src/lib/third_party/include @CFLAGS@ @CUSTOM_NDPI@ -shared -#LIBNDPI=../src/lib/libndpi.so.@NDPI_VERSION_SHORT@ -LIBNDPI=../src/lib/libndpi.a -LDFLAGS=$(CFILE) $(LIBNDPI) -lpcap @LIBS@ @ADDITIONAL_LIBS@ -SHARE = -soname,ndpi_wrap + +BUILD_SRC=../src +ROOT_SRC=@srcdir@/../src +ROOT_PYTHON=@srcdir@ + +PYTHON_EXEC?=python +CFLAGS=-I$(ROOT_PYTHON) -I$(ROOT_SRC)/include -I$(BUILD_SRC)/include -I$(ROOT_SRC)/lib/third_party/include \ + @CFLAGS@ @CUSTOM_NDPI@ -shared +LIBNDPI=$(BUILD_SRC)/.libs/libndpi.a +LDFLAGS=$(LIBNDPI) -lpcap @LIBS@ +SHARE=-soname,ndpi_wrap SO=ndpi_wrap.so -OBJS = ndpi_wrap.o -CFILE = ndpi_wrap.c -PIC = -fPIC -PREFIX?=/usr/local +CFILE=$(ROOT_PYTHON)/ndpi_wrap.c +PIC=-fPIC -DPIC +PREFIX?=@prefix@ .PHONY: all UNAME_S := $(shell uname -s) -ifeq ($(UNAME_S),Darwin)#do something for linux - SHARE=-install_name,ndpiReader.so +ifeq ($(UNAME_S),Darwin) +SHARE=-install_name,ndpiReader.so endif all: $(SO) $(SO): $(CFILE) $(LIBNDPI) Makefile - $(CC) $(CFLAGS) -Wl,$(SHARE) -o $@ $(PIC) $(LDFLAGS) -# ln -s $(LIBNDPI) . + $(CC) $(CFLAGS) -Wl,$(SHARE) $(CFILE) -o $@ $(PIC) $(LDFLAGS) + +example: $(SO) + env NDPI_WRAP="$(realpath ./ndpi_wrap.so)" $(PYTHON_EXEC) $(ROOT_PYTHON)/ndpi_example.py clean: /bin/rm -f $(SO) diff --git a/python/ndpi_example.py b/python/ndpi_example.py index d134d3947..7b932941d 100755 --- a/python/ndpi_example.py +++ b/python/ndpi_example.py @@ -100,6 +100,8 @@ lista = [] # used to avoid impropriate memory deallocation from python # check ndpi version if ndpi.ndpi_get_api_version() != ndpi.ndpi_wrap_get_api_version(): print("nDPI Library version mismatch: please make sure this code and the nDPI library are in sync\n") + print("libnDPI api version..: " + str(ndpi.ndpi_get_api_version())) + print("NDPI_WRAP api version: " + str(ndpi.ndpi_wrap_get_api_version())) sys.exit(-1) # create data structure of ndpi diff --git a/python/ndpi_typestruct.py b/python/ndpi_typestruct.py index 1944a5f3c..4ccd15818 100644 --- a/python/ndpi_typestruct.py +++ b/python/ndpi_typestruct.py @@ -21,7 +21,14 @@ If not, see . from ctypes import CDLL, Structure, c_uint16, c_int, c_ulong, c_uint32, CFUNCTYPE, c_void_p, POINTER, c_char_p, c_uint8 from ctypes import c_char, c_uint, c_int16, c_longlong, c_size_t, Union, c_ubyte, c_uint64, c_int32, c_ushort, cast from os.path import abspath, dirname -ndpi = CDLL(dirname(abspath(__file__)) + '/ndpi_wrap.so') + +try: + ndpi = CDLL(dirname(abspath(__file__)) + '/ndpi_wrap.so') +except OSError: + from os import getenv + if getenv('NDPI_WRAP') is None: + raise RuntimeError('Environment variable NDPI_WRAP not set and default path does not exist: ' + dirname(abspath(__file__)) + '/ndpi_wrap.so') + ndpi = CDLL(getenv('NDPI_WRAP')) # ----------------------------------------------- Structures ----------------------------------------------------------- @@ -94,7 +101,7 @@ class NDPIProtoDefaultsT(Structure): _fields_ = [ ("protoName", c_char_p), ("protoCategory", c_uint), - ("subprotocols", c_uint16_p), + ("subprotocols", POINTER(c_uint16)), ("subprotocol_count", c_uint32), ("protoId", c_uint16), ("protoIdx", c_uint16), diff --git a/src/Makefile.am b/src/Makefile.am new file mode 100644 index 000000000..f11b7032e --- /dev/null +++ b/src/Makefile.am @@ -0,0 +1,248 @@ +EXTRA_DIST = \ + lib/third_party/src/hll/hll.c \ + lib/third_party/src/hll/MurmurHash3.c \ + lib/ndpi_content_match.c.inc \ + lib/protocols/btlib.h \ + lib/third_party/include/ahocorasick.h \ + lib/third_party/include/hll.h \ + lib/third_party/include/libcache.h \ + lib/third_party/include/libinjection.h \ + lib/third_party/include/libinjection_html5.h \ + lib/third_party/include/libinjection_sqli_data.h \ + lib/third_party/include/libinjection_sqli.h \ + lib/third_party/include/libinjection_xss.h \ + lib/third_party/include/MurmurHash3.h \ + lib/third_party/include/ndpi_md5.h \ + lib/third_party/include/ndpi_patricia.h \ + lib/third_party/include/ndpi_sha1.h \ + lib/third_party/include/rce_injection.h \ + lib/third_party/include/uthash.h + +includendpidir = $(includedir)/ndpi + +includendpi_HEADERS = \ + include/ndpi_api.h \ + include/ndpi_config.h \ + include/ndpi_define.h \ + include/ndpi_classify.h \ + include/ndpi_encryption.h \ + include/ndpi_includes.h \ + include/ndpi_includes_OpenBSD.h \ + include/ndpi_main.h \ + include/ndpi_patricia_typedefs.h \ + include/ndpi_protocol_ids.h \ + include/ndpi_protocols.h \ + include/ndpi_typedefs.h \ + include/ndpi_unix.h \ + include/ndpi_utils.h \ + include/ndpi_win32.h + +lib_LTLIBRARIES = libndpi.la + +libndpi_la_SOURCES = \ + lib/ndpi_analyze.c \ + lib/ndpi_classify.c \ + lib/ndpi_community_id.c \ + lib/ndpi_geoip.c \ + lib/ndpi_main.c \ + lib/ndpi_serializer.c \ + lib/ndpi_utils.c \ + lib/protocols/afp.c \ + lib/protocols/aimini.c \ + lib/protocols/ajp.c \ + lib/protocols/amazon_video.c \ + lib/protocols/among_us.c \ + lib/protocols/amqp.c \ + lib/protocols/applejuice.c \ + lib/protocols/apple_push.c \ + lib/protocols/armagetron.c \ + lib/protocols/attic/flash.c \ + lib/protocols/attic/ftp.c \ + lib/protocols/attic/manolito.c \ + lib/protocols/attic/popo.c \ + lib/protocols/attic/secondlife.c \ + lib/protocols/avast_securedns.c \ + lib/protocols/ayiya.c \ + lib/protocols/bgp.c \ + lib/protocols/bittorrent.c \ + lib/protocols/bjnp.c \ + lib/protocols/btlib.c \ + lib/protocols/capwap.c \ + lib/protocols/checkmk.c \ + lib/protocols/ciscovpn.c \ + lib/protocols/citrix.c \ + lib/protocols/coap.c \ + lib/protocols/collectd.c \ + lib/protocols/corba.c \ + lib/protocols/cpha.c \ + lib/protocols/crossfire.c \ + lib/protocols/csgo.c \ + lib/protocols/dcerpc.c \ + lib/protocols/dhcp.c \ + lib/protocols/dhcpv6.c \ + lib/protocols/diameter.c \ + lib/protocols/directconnect.c \ + lib/protocols/directdownloadlink.c \ + lib/protocols/dnp3.c \ + lib/protocols/dns.c \ + lib/protocols/dnscrypt.c \ + lib/protocols/dofus.c \ + lib/protocols/drda.c \ + lib/protocols/dropbox.c \ + lib/protocols/eaq.c \ + lib/protocols/edonkey.c \ + lib/protocols/fasttrack.c \ + lib/protocols/fiesta.c \ + lib/protocols/fix.c \ + lib/protocols/florensia.c \ + lib/protocols/ftp_control.c \ + lib/protocols/ftp_data.c \ + lib/protocols/genshin_impact.c \ + lib/protocols/git.c \ + lib/protocols/gnutella.c \ + lib/protocols/gtp.c \ + lib/protocols/guildwars.c \ + lib/protocols/h323.c \ + lib/protocols/halflife2_and_mods.c \ + lib/protocols/hangout.c \ + lib/protocols/hpvirtgrp.c \ + lib/protocols/http.c \ + lib/protocols/iax.c \ + lib/protocols/icecast.c \ + lib/protocols/iec60870-5-104.c \ + lib/protocols/imo.c \ + lib/protocols/ipp.c \ + lib/protocols/irc.c \ + lib/protocols/jabber.c \ + lib/protocols/kakaotalk_voice.c \ + lib/protocols/kerberos.c \ + lib/protocols/kontiki.c \ + lib/protocols/ldap.c \ + lib/protocols/lisp.c \ + lib/protocols/lotus_notes.c \ + lib/protocols/mail_imap.c \ + lib/protocols/mail_pop.c \ + lib/protocols/mail_smtp.c \ + lib/protocols/maplestory.c \ + lib/protocols/megaco.c \ + lib/protocols/memcached.c \ + lib/protocols/mgcp.c \ + lib/protocols/mining.c \ + lib/protocols/modbus.c \ + lib/protocols/mongodb.c \ + lib/protocols/mpegts.c \ + lib/protocols/mqtt.c \ + lib/protocols/mssql_tds.c \ + lib/protocols/mysql.c \ + lib/protocols/nats.c \ + lib/protocols/nest_log_sink.c \ + lib/protocols/netbios.c \ + lib/protocols/netflow.c \ + lib/protocols/nfs.c \ + lib/protocols/nintendo.c \ + lib/protocols/noe.c \ + lib/protocols/non_tcp_udp.c \ + lib/protocols/ntp.c \ + lib/protocols/ookla.c \ + lib/protocols/openft.c \ + lib/protocols/openvpn.c \ + lib/protocols/oracle.c \ + lib/protocols/postgres.c \ + lib/protocols/ppstream.c \ + lib/protocols/pptp.c \ + lib/protocols/qq.c \ + lib/protocols/quic.c \ + lib/protocols/radius.c \ + lib/protocols/rdp.c \ + lib/protocols/redis_net.c \ + lib/protocols/rsync.c \ + lib/protocols/rtcp.c \ + lib/protocols/rtmp.c \ + lib/protocols/rtp.c \ + lib/protocols/rtsp.c \ + lib/protocols/rx.c \ + lib/protocols/s7comm.c \ + lib/protocols/sflow.c \ + lib/protocols/shoutcast.c \ + lib/protocols/sip.c \ + lib/protocols/skinny.c \ + lib/protocols/skype.c \ + lib/protocols/smb.c \ + lib/protocols/smpp.c \ + lib/protocols/snmp_proto.c \ + lib/protocols/soap.c \ + lib/protocols/socks45.c \ + lib/protocols/someip.c \ + lib/protocols/sopcast.c \ + lib/protocols/soulseek.c \ + lib/protocols/spotify.c \ + lib/protocols/ssdp.c \ + lib/protocols/ssh.c \ + lib/protocols/starcraft.c \ + lib/protocols/stealthnet.c \ + lib/protocols/steam.c \ + lib/protocols/stun.c \ + lib/protocols/syslog.c \ + lib/protocols/targus_getdata.c \ + lib/protocols/tcp_udp.c \ + lib/protocols/teamspeak.c \ + lib/protocols/teamviewer.c \ + lib/protocols/telegram.c \ + lib/protocols/telnet.c \ + lib/protocols/teredo.c \ + lib/protocols/tftp.c \ + lib/protocols/thunder.c \ + lib/protocols/tinc.c \ + lib/protocols/tls.c \ + lib/protocols/tvuplayer.c \ + lib/protocols/ubntac2.c \ + lib/protocols/usenet.c \ + lib/protocols/vhua.c \ + lib/protocols/viber.c \ + lib/protocols/vmware.c \ + lib/protocols/vnc.c \ + lib/protocols/warcraft3.c \ + lib/protocols/websocket.c \ + lib/protocols/whatsapp.c \ + lib/protocols/whoisdas.c \ + lib/protocols/wireguard.c \ + lib/protocols/world_of_kung_fu.c \ + lib/protocols/world_of_warcraft.c \ + lib/protocols/wsd.c \ + lib/protocols/xbox.c \ + lib/protocols/xdmcp.c \ + lib/protocols/z3950.c \ + lib/protocols/zabbix.c \ + lib/protocols/zattoo.c \ + lib/protocols/zeromq.c \ + lib/third_party/src/ahocorasick.c \ + lib/third_party/src/libcache.c \ + lib/third_party/src/libinjection_html5.c \ + lib/third_party/src/libinjection_sqli.c \ + lib/third_party/src/libinjection_xss.c \ + lib/third_party/src/ndpi_md5.c \ + lib/third_party/src/ndpi_patricia.c \ + lib/third_party/src/ndpi_sha1.c \ + lib/third_party/src/sha1-fast.c \ + lib/third_party/src/strptime.c + +libndpi_la_CFLAGS = \ + -fPIC -DPIC \ + -I$(top_srcdir)/src/include \ + -I$(top_srcdir)/src/lib \ + -I$(top_srcdir)/src/lib/third_party/include \ + -DNDPI_LIB_COMPILATION \ + -Wall \ + @CUSTOM_NDPI@ + +# Remember that libtool's semantic versioning is different from nDPI's versioning! +# See: https://www.gnu.org/software/libtool/manual/html_node/Libtool-versioning.html +libndpi_la_LDFLAGS = -version-info @NDPI_MAJOR@:@NDPI_MINOR@:@NDPI_PATCH@ + +check: + cppcheck --template='{file}:{line}:{severity}:{message}' --quiet --enable=all --force \ + -Iinclude \ + -I$(top_srcdir)/src/include \ + -I$(top_srcdir)/src/lib \ + -I$(top_srcdir)/src/lib/third_party/include \ + $(top_srcdir)/src/lib/*.c $(top_srcdir)/src/lib/protocols/*. diff --git a/src/include/ndpi_utils.h b/src/include/ndpi_utils.h index 983aae283..8dcd5ac77 100644 --- a/src/include/ndpi_utils.h +++ b/src/include/ndpi_utils.h @@ -4,6 +4,7 @@ #define __NDPI_UTILS_H__ #include "ndpi_define.h" +#include "ndpi_includes.h" #define MYDBG(m, ...) \ printf(" DBG[%s:%s:%u]: \t" m "\n", __FILE__, __FUNCTION__, __LINE__, ##__VA_ARGS__); diff --git a/src/lib/Makefile.in b/src/lib/Makefile.in deleted file mode 100644 index ef866fe96..000000000 --- a/src/lib/Makefile.in +++ /dev/null @@ -1,82 +0,0 @@ -# -# Simple non-autotools dependent makefile -# -# ./autogen.sh -# cd src/lib -# make Makefile -# - -# -# Installation directories -# -prefix = @prefix@ -libdir = ${prefix}/lib -includedir = ${prefix}/include/ndpi -CC = @GNU_PREFIX@@CC@ -CFLAGS += -fPIC -DPIC -I../include -Ithird_party/include -DNDPI_LIB_COMPILATION -Wall @CFLAGS@ @CUSTOM_NDPI@ -LDFLAGS = @LDFLAGS@ @ADDITIONAL_LIBS@ @LIBS@ -RANLIB = @GNU_PREFIX@ranlib - -OBJECTS = $(patsubst protocols/%.c, protocols/%.o, $(wildcard protocols/*.c)) $(patsubst third_party/src/%.c, third_party/src/%.o, $(wildcard third_party/src/*.c)) $(patsubst ./%.c, ./%.o, $(wildcard ./*.c)) -HEADERS = $(wildcard ../include/*.h) -NDPI_VERSION_MAJOR = @NDPI_MAJOR@ -NDPI_LIB_STATIC = libndpi.a -NDPI_LIB_SHARED_BASE = libndpi.so -NDPI_LIB_SHARED = $(NDPI_LIB_SHARED_BASE).@NDPI_VERSION_SHORT@ -NDPI_LIBS = $(NDPI_LIB_STATIC) $(NDPI_LIB_SHARED) - -ifneq ($(OS),Windows_NT) -OS := $(shell uname) -endif -BUILD_MINGW = @BUILD_MINGW@ - -ifeq ($(OS),Darwin) -CC=clang -SONAME_FLAG= -else -ifneq ($(BUILD_MINGW),) -NDPI_LIB_SHARED_BASE = libndpi -NDPI_LIB_SHARED = $(NDPI_LIB_SHARED_BASE)-@NDPI_VERSION_SHORT@.dll -else -SONAME_FLAG=-Wl,-soname,$(NDPI_LIB_SHARED_BASE).$(NDPI_VERSION_MAJOR) -endif -endif - -all: $(NDPI_LIBS) - -ndpi_main.c: ndpi_content_match.c.inc - -$(NDPI_LIB_STATIC): $(OBJECTS) - @GNU_PREFIX@ar rc $@ $(OBJECTS) - $(RANLIB) $@ - -$(NDPI_LIB_SHARED): $(OBJECTS) - $(CC) -shared -fPIC $(SONAME_FLAG) -o $@ $(OBJECTS) $(LDFLAGS) - ln -fs $(NDPI_LIB_SHARED) $(NDPI_LIB_SHARED_BASE) - ln -fs $(NDPI_LIB_SHARED) $(NDPI_LIB_SHARED_BASE).$(NDPI_VERSION_MAJOR) - -%.o: %.c $(HEADERS) Makefile - $(CC) $(CFLAGS) -c $< -o $@ - -clean: - /bin/rm -f $(NDPI_LIB_STATIC) $(OBJECTS) *.o *.so *.lo libndpi.so* - -distdir: - cp ndpi_content_match.c.inc '$(distdir)/' - find . -type d | xargs -I'{}' mkdir -p '$(distdir)/{}' - find ../include -type f -name '*.h' | xargs -I'{}' cp '{}' '$(distdir)/{}' - find . -type f -name '*.c' -o -name '*.h' | xargs -I'{}' cp '{}' '$(distdir)/{}' - -distclean: clean - /bin/rm -f Makefile - -cppcheck: - cppcheck --template='{file}:{line}:{severity}:{message}' --quiet --enable=all --force -I ../include *.c protocols/*.c - -install: $(NDPI_LIBS) - mkdir -p $(DESTDIR)$(libdir) - cp $(NDPI_LIBS) $(DESTDIR)$(libdir)/ - cp -P $(NDPI_LIB_SHARED_BASE) $(DESTDIR)$(libdir)/ - cp -P $(NDPI_LIB_SHARED_BASE).$(NDPI_VERSION_MAJOR) $(DESTDIR)$(libdir)/ - mkdir -p $(DESTDIR)$(includedir) - cp ../include/*.h $(DESTDIR)$(includedir) diff --git a/tests/Makefile.am b/tests/Makefile.am index 271516b87..b6d37ddf5 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -1,6 +1,6 @@ -TESTS = do.sh +TESTS = do.sh do-unit.sh -EXTRA_DIST = ossfuzz.sh pcap result +EXTRA_DIST = do_valgrind.sh do-dga.sh ossfuzz.sh pcap result all: @echo -n "" diff --git a/tests/dga/Makefile.in b/tests/dga/Makefile.in index fc94ed7e7..aa2fa5d86 100644 --- a/tests/dga/Makefile.in +++ b/tests/dga/Makefile.in @@ -1,25 +1,24 @@ CC=@CC@ CXX=@CXX@ -SRCHOME=../../src - -CFLAGS=-g -fPIC -DPIC -I$(SRCHOME)/include @JSONC_CFLAGS@ @PCAP_INC@ @CFLAGS@ -LIBNDPI=$(SRCHOME)/lib/libndpi.a -LDFLAGS=$(LIBNDPI) @PCAP_LIB@ @LIBS@ @ADDITIONAL_LIBS@ @JSONC_LIBS@ -lpthread @LDFLAGS@ -HEADERS=$(SRCHOME)/include/ndpi_api.h $(SRCHOME)/include/ndpi_typedefs.h $(SRCHOME)/include/ndpi_protocol_ids.h +BUILD_SRC=../../src +ROOT_SRC=@srcdir@/../../src +ROOT_TESTS_DGA=@srcdir@ + +CFLAGS=-g -fPIC -DPIC -I$(ROOT_SRC)/include -I$(BUILD_SRC)/include @JSONC_CFLAGS@ @PCAP_INC@ @CFLAGS@ +LIBNDPI=$(BUILD_SRC)/.libs/libndpi.a +LDFLAGS=$(LIBNDPI) @PCAP_LIB@ @LIBS@ @JSONC_LIBS@ -lpthread @LDFLAGS@ +HEADERS=$(BUILD_SRC)/include/ndpi_api.h $(ROOT_SRC)/include/ndpi_typedefs.h $(ROOT_SRC)/include/ndpi_protocol_ids.h OBJS=dga_evaluate PREFIX?=@prefix@ all: dga_evaluate -EXECUTABLE_SOURCES := dga_evaluate.c -COMMON_SOURCES := $(filter-out $(EXECUTABLE_SOURCES),$(wildcard *.c )) - dga_evaluate: $(LIBNDPI) dga_evaluate.o $(CC) $(CFLAGS) dga_evaluate.o -o $@ $(LDFLAGS) -%.o: %.c $(HEADERS) Makefile - $(CC) $(CFLAGS) -c $< -o $@ +%.o: $(ROOT_TESTS_DGA)/%.c $(HEADERS) Makefile + $(CC) $(CFLAGS) -c $< -o $(notdir $@) clean: /bin/rm -f *.o dga_evaluate @@ -30,7 +29,12 @@ install: echo "" distdir: - + cp \ + $(ROOT_TESTS_DGA)/dga_evaluate.c \ + $(ROOT_TESTS_DGA)/README.md \ + $(ROOT_TESTS_DGA)/test_dga.csv \ + $(ROOT_TESTS_DGA)/test_non_dga.csv \ + '$(distdir)/' distclean: clean /bin/rm -f Makefile diff --git a/tests/do-dga.sh b/tests/do-dga.sh deleted file mode 100755 index 93408a012..000000000 --- a/tests/do-dga.sh +++ /dev/null @@ -1,65 +0,0 @@ -#!/bin/sh - -cd "$(dirname "${0}")" - -# Baseline performances ------------------------------------------------------------------------------------------------ -# Important notes: BASE values must be integers examples and represents percentage (e.g. 79%, 98%). -BASE_ACCURACY=69 -BASE_PRECISION=89 -BASE_RECALL=41 -# ---------------------------------------------------------------------------------------------------------------------- - -DGA_EVALUATE="./dga/dga_evaluate" -DGA_DATA="dga/test_dga.csv" -NON_DGA_DATA="dga/test_non_dga.csv" -DGA_DATA_SIZE=0 -NON_DGA_DATA_SIZE=0 -DATA_SIZE=0 -RC=0 - -get_evaluation_data_size() { - DGA_DATA_SIZE=`wc -l dga/test_dga.csv | awk '{split($0,a," "); print a[1]}'` - NON_DGA_DATA_SIZE=`wc -l dga/test_non_dga.csv | awk '{split($0,a," "); print a[1]}'` - DATA_SIZE=$(( $NON_DGA_DATA_SIZE + $DGA_DATA_SIZE )) -} - -evaluate_ndpi_dga_detection() { - # DGA detection is a binary classification problem, We evaluate the following metrics: - # Accuracy: (TP + TN) / (TP + TN + FN + FP) - # Precision: TP / (TP + FP) - # Recall: TP / (TP + FN) - - TP=`$DGA_EVALUATE dga/test_dga.csv` - FN=$(( $DGA_DATA_SIZE - $TP )) - FP=`$DGA_EVALUATE dga/test_non_dga.csv` - TN=$(( $NON_DGA_DATA_SIZE - $FP )) - - ACCURACY=`echo "print(int(((${TP} + ${TN})/(${TP} + ${TN} + ${FP} + ${FN}))*100))" | python3` - PRECISION=`echo "print(int(((${TP})/(${TP} + ${FP}))*100))" | python3` - RECALL=`echo "print(int(((${TP})/(${TP} + ${FN}))*100))" | python3` - - # In case modified version of classification algorithm decreases performances, test do not pass. - if [ $ACCURACY -lt $BASE_ACCURACY ]; then - printf "ERROR: Your modifications decreased DGA classifier accuracy: 0.${BASE_ACCURACY} decreased to 0.${ACCURACY}!\n" - RC=1 - fi - if [ $PRECISION -lt $BASE_PRECISION ]; then - printf "ERROR: Your modifications decreased DGA classifier precision: 0.${BASE_PRECISION} decreased to 0.${PRECISION}!\n" - RC=1 - fi - if [ $RECALL -lt $BASE_RECALL ]; then - printf "ERROR: Your modifications decreased DGA classifier recall: 0.${BASE_RECALL} decreased to 0.${RECALL}!\n" - RC=1 - fi - - # Finally we print the current performances, upgrade BASE_ metrics in case you improved it. - echo "DGA detection performances report:" - echo "Accuracy=0.$ACCURACY" - echo "Precision=0.$PRECISION" - echo "Recall=0.$RECALL" -} - -get_evaluation_data_size -evaluate_ndpi_dga_detection - -exit $RC diff --git a/tests/do-dga.sh.in b/tests/do-dga.sh.in new file mode 100755 index 000000000..4c5bd8766 --- /dev/null +++ b/tests/do-dga.sh.in @@ -0,0 +1,65 @@ +#!/bin/sh + +cd "$(dirname "${0}")" + +# Baseline performances ------------------------------------------------------------------------------------------------ +# Important notes: BASE values must be integers examples and represents percentage (e.g. 79%, 98%). +BASE_ACCURACY=69 +BASE_PRECISION=89 +BASE_RECALL=41 +# ---------------------------------------------------------------------------------------------------------------------- + +DGA_EVALUATE="dga/dga_evaluate" +DGA_DATA="@srcdir@/dga/test_dga.csv" +NON_DGA_DATA="@srcdir@/dga/test_non_dga.csv" +DGA_DATA_SIZE=0 +NON_DGA_DATA_SIZE=0 +DATA_SIZE=0 +RC=0 + +get_evaluation_data_size() { + DGA_DATA_SIZE=`wc -l ${DGA_DATA} | awk '{split($0,a," "); print a[1]}'` + NON_DGA_DATA_SIZE=`wc -l ${NON_DGA_DATA} | awk '{split($0,a," "); print a[1]}'` + DATA_SIZE=$(( $NON_DGA_DATA_SIZE + $DGA_DATA_SIZE )) +} + +evaluate_ndpi_dga_detection() { + # DGA detection is a binary classification problem, We evaluate the following metrics: + # Accuracy: (TP + TN) / (TP + TN + FN + FP) + # Precision: TP / (TP + FP) + # Recall: TP / (TP + FN) + + TP=`$DGA_EVALUATE ${DGA_DATA}` + FN=$(( $DGA_DATA_SIZE - $TP )) + FP=`$DGA_EVALUATE ${NON_DGA_DATA}` + TN=$(( $NON_DGA_DATA_SIZE - $FP )) + + ACCURACY=`echo "print(int(((${TP} + ${TN})/(${TP} + ${TN} + ${FP} + ${FN}))*100))" | python3` + PRECISION=`echo "print(int(((${TP})/(${TP} + ${FP}))*100))" | python3` + RECALL=`echo "print(int(((${TP})/(${TP} + ${FN}))*100))" | python3` + + # In case modified version of classification algorithm decreases performances, test do not pass. + if [ $ACCURACY -lt $BASE_ACCURACY ]; then + printf "ERROR: Your modifications decreased DGA classifier accuracy: 0.${BASE_ACCURACY} decreased to 0.${ACCURACY}!\n" + RC=1 + fi + if [ $PRECISION -lt $BASE_PRECISION ]; then + printf "ERROR: Your modifications decreased DGA classifier precision: 0.${BASE_PRECISION} decreased to 0.${PRECISION}!\n" + RC=1 + fi + if [ $RECALL -lt $BASE_RECALL ]; then + printf "ERROR: Your modifications decreased DGA classifier recall: 0.${BASE_RECALL} decreased to 0.${RECALL}!\n" + RC=1 + fi + + # Finally we print the current performances, upgrade BASE_ metrics in case you improved it. + echo "DGA detection performances report:" + echo "Accuracy=0.$ACCURACY" + echo "Precision=0.$PRECISION" + echo "Recall=0.$RECALL" +} + +get_evaluation_data_size +evaluate_ndpi_dga_detection + +exit $RC diff --git a/tests/do-unit.sh b/tests/do-unit.sh deleted file mode 100755 index 937be26ff..000000000 --- a/tests/do-unit.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/sh - -cd "$(dirname "${0}")" - -UNIT="./unit/unit" - -RC=0 - -check_unit() { - - case "$CXXFLAGS" in - # Skipping tests with sanitizer enabled due to use-of-uninitialized-value in json-c - *sanitize* ) - echo "Skipping unit tests for this environment" - return - ;; - * ) - echo "" - echo "Running unit tests.." - ;; - esac - - $UNIT - UNIT_RC=$? - if [ $UNIT_RC -ne 0 ]; then - RC=1 - fi -} - -check_unit - -exit $RC diff --git a/tests/do-unit.sh.in b/tests/do-unit.sh.in new file mode 100755 index 000000000..cefc8a9ca --- /dev/null +++ b/tests/do-unit.sh.in @@ -0,0 +1,20 @@ +#!/bin/sh + +cd "$(dirname "${0}")" + +UNIT="./unit/unit" + +RC=0 + +check_unit() { + + $UNIT + UNIT_RC=$? + if [ $UNIT_RC -ne 0 ]; then + RC=1 + fi +} + +check_unit + +exit $RC diff --git a/tests/do.sh.in b/tests/do.sh.in index 01fad8497..4c081773b 100755 --- a/tests/do.sh.in +++ b/tests/do.sh.in @@ -1,26 +1,30 @@ #!/bin/sh -cd "$(dirname "${0}")" +BUILD_TESTS="$(realpath $(dirname "${0}"))" +cd "$(dirname "${0}")/@srcdir@" FUZZY_TESTING_ENABLED=@BUILD_FUZZTARGETS@ GCRYPT_ENABLED=@GCRYPT_ENABLED@ GCRYPT_PCAPS="gquic.pcap quic-23.pcap quic-24.pcap quic-27.pcap quic-28.pcap quic-29.pcap quic-mvfst-22.pcap quic-mvfst-27.pcapng quic-mvfst-exp.pcap quic_q50.pcap quic_t50.pcap quic_t51.pcap quic_0RTT.pcap quic_interop_V.pcapng quic-33.pcapng doq.pcapng doq_adguard.pcapng dlt_ppp.pcap os_detected.pcapng quic_frags_ch_out_of_order_same_packet_craziness.pcapng quic_frags_ch_in_multiple_packets.pcapng" -READER="../example/ndpiReader -p ../example/protos.txt -c ../example/categories.txt -r ../example/risky_domains.txt -j ../example/ja3_fingerprints.csv -S ../example/sha1_fingerprints.csv" +READER="${BUILD_TESTS}/../example/ndpiReader -p ../example/protos.txt -c ../example/categories.txt -r ../example/risky_domains.txt -j ../example/ja3_fingerprints.csv -S ../example/sha1_fingerprints.csv" RC=0 PCAPS=`cd pcap; /bin/ls *.pcap *.pcapng` -if [ ! -x "../example/ndpiReader" ]; then - echo "$0: Missing $(realpath ../example/ndpiReader)" - echo "$0: Run ./configure and make first" +if [ ! -x "${BUILD_TESTS}/../example/ndpiReader" ]; then + echo "$0: Missing ${BUILD_TESTS}/../example/ndpiReader)" + echo "$0: Run make first." exit 1 fi fuzzy_testing() { - if [ -f ../fuzz/fuzz_ndpi_reader ]; then - ../fuzz/fuzz_ndpi_reader -max_total_time="${MAX_TOTAL_TIME:-592}" -print_pcs=1 -workers="${FUZZY_WORKERS:-0}" -jobs="${FUZZY_JOBS:-0}" pcap/ - fi + if [ -f "${BUILD_TESTS}/../fuzz/fuzz_ndpi_reader" ]; then + ${BUILD_TESTS}/../fuzz/fuzz_ndpi_reader \ + -max_total_time="${MAX_TOTAL_TIME:-592}" -print_pcs=1 \ + -workers="${FUZZY_WORKERS:-0}" -jobs="${FUZZY_JOBS:-0}" \ + pcap/ + fi } build_results() { diff --git a/tests/do_valgrind.sh.in b/tests/do_valgrind.sh.in index 8ab0509a8..c76cc6e1a 100755 --- a/tests/do_valgrind.sh.in +++ b/tests/do_valgrind.sh.in @@ -1,17 +1,18 @@ #!/bin/sh -cd "$(dirname "${0}")" +BUILD_TESTS="$(realpath $(dirname "${0}"))" +cd "$(dirname "${0}")/@srcdir@" GCRYPT_ENABLED=@GCRYPT_ENABLED@ GCRYPT_PCAPS="gquic.pcap quic-23.pcap quic-24.pcap quic-27.pcap quic-28.pcap quic-29.pcap quic-mvfst-22.pcap quic-mvfst-27.pcap quic-mvfst-exp.pcap quic_q50.pcap quic_t50.pcap quic_t51.pcap quic_0RTT.pcap quic_interop_V.pcapng quic-33.pcapng doq.pcapng doq_adguard.pcapng dlt_ppp.pcap" -READER="valgrind -q --leak-check=full ../example/ndpiReader -p ../example/protos.txt -c ../example/categories.txt" +READER="valgrind -q --leak-check=full ${BUILD_TESTS}/../example/ndpiReader -p ../example/protos.txt -c ../example/categories.txt" RC=0 PCAPS=`cd pcap; /bin/ls *.pcap` -if [ ! -x "../example/ndpiReader" ]; then - echo "$0: Missing $(realpath ../example/ndpiReader)" - echo "$0: Run ./configure and make first" +if [ ! -x "${BUILD_TESTS}/../example/ndpiReader" ]; then + echo "$0: Missing $(realpath ${BUILD_TESTS}/../example/ndpiReader)" + echo "$0: Run make first." exit 1 fi diff --git a/tests/unit/Makefile.in b/tests/unit/Makefile.in index efbd387c8..808a177dd 100644 --- a/tests/unit/Makefile.in +++ b/tests/unit/Makefile.in @@ -1,25 +1,24 @@ CC=@CC@ CXX=@CXX@ -SRCHOME=../../src - -CFLAGS=-g -fPIC -DPIC -I$(SRCHOME)/include @JSONC_CFLAGS@ @PCAP_INC@ @CFLAGS@ -LIBNDPI=$(SRCHOME)/lib/libndpi.a -LDFLAGS=$(LIBNDPI) @PCAP_LIB@ @LIBS@ @ADDITIONAL_LIBS@ @JSONC_LIBS@ -lpthread @LDFLAGS@ -HEADERS=$(SRCHOME)/include/ndpi_api.h $(SRCHOME)/include/ndpi_typedefs.h $(SRCHOME)/include/ndpi_protocol_ids.h +BUILD_SRC=../../src +ROOT_SRC=@srcdir@/../../src +ROOT_TESTS_UNIT=@srcdir@ + +CFLAGS=-g -fPIC -DPIC -I$(ROOT_SRC)/include -I$(BUILD_SRC)/include @JSONC_CFLAGS@ @PCAP_INC@ @CFLAGS@ +LIBNDPI=$(BUILD_SRC)/.libs/libndpi.a +LDFLAGS=$(LIBNDPI) @PCAP_LIB@ @LIBS@ @JSONC_LIBS@ -lpthread @LDFLAGS@ +HEADERS=$(BUILD_SRC)/include/ndpi_api.h $(ROOT_SRC)/include/ndpi_typedefs.h $(ROOT_SRC)/include/ndpi_protocol_ids.h OBJS=unit PREFIX?=@prefix@ all: unit -EXECUTABLE_SOURCES := unit.c -COMMON_SOURCES := $(filter-out $(EXECUTABLE_SOURCES),$(wildcard *.c )) - unit: $(LIBNDPI) unit.o $(CC) $(CFLAGS) unit.o -o $@ $(LDFLAGS) -%.o: %.c $(HEADERS) Makefile - $(CC) $(CFLAGS) -c $< -o $@ +%.o: $(ROOT_TESTS_UNIT)/%.c $(HEADERS) Makefile + $(CC) $(CFLAGS) -c $< -o $(notdir $@) clean: /bin/rm -f *.o unit @@ -30,7 +29,9 @@ install: echo "" distdir: - + cp \ + $(ROOT_TESTS_UNIT)/unit.c \ + '$(distdir)/' distclean: clean /bin/rm -f Makefile diff --git a/utils/generate_automake_am.sh b/utils/generate_automake_am.sh new file mode 100755 index 000000000..5219a6445 --- /dev/null +++ b/utils/generate_automake_am.sh @@ -0,0 +1,164 @@ +#!/usr/bin/env sh +# +# This script is used for automatic generation of src/Makefile.am and fuzz/Makefile.am +# That way it is not necessary anymore to use wildcard *.c includes. +# Automake does not support it: https://www.gnu.org/software/automake/manual/html_node/Wildcards.html +# +# This script should run everytime a new *.c file should be part of the library. +# + +set -e + +MYDIR="$(realpath "$(dirname "${0}")")" + +generate_src_am() +{ + # enumerate all *.c sources below src/ with a max. depth of three + cd "${MYDIR}/../src" + C_INCLUDES_GEN=$(find include -iname "*.h.in" | sort) + C_INCLUDES_PRV=$(find lib lib/protocols lib/third_party/include -maxdepth 1 -iname "*.h" -o -iname "*.c.inc" | sort) + C_INCLUDES=$(find include -iname "*.h" | sort) + C_SOURCES=$(find lib -maxdepth 3 -iname "*.c" | sort) + C_THIRD_PARTY_HLL_SOURCES=$(find lib/third_party/src/hll -iname "*.c" | sort) + + # extra dist source files (required by nDPI through #include) + printf '%s' 'EXTRA_DIST =' >Makefile.am + + # HLL sources + for src in ${C_THIRD_PARTY_HLL_SOURCES}; do + printf ' \\\n\t%s' "${src}" >>Makefile.am + done + + # private headers + for inc in ${C_INCLUDES_PRV}; do + printf ' \\\n\t%s' "${inc}" >>Makefile.am + done + printf '\n\n' >>Makefile.am + + printf '%s\n\n' 'includendpidir = $(includedir)/ndpi' >>Makefile.am + + printf '%s' 'includendpi_HEADERS =' >>Makefile.am + + # generated headers + for inc in ${C_INCLUDES_GEN}; do + printf ' \\\n\t%s' "${inc%.in}" >>Makefile.am + done + + # headers + for inc in ${C_INCLUDES}; do + SKIP_INC=0 + for check_inc in ${C_INCLUDES_GEN}; do + test "x${inc}.in" != "x${check_inc}" || SKIP_INC=1 + done + test ${SKIP_INC} -eq 0 || continue + printf ' \\\n\t%s' "${inc}" >>Makefile.am + done + printf '\n\n' >>Makefile.am + + # libtool init + printf '%s\n\n' 'lib_LTLIBRARIES = libndpi.la' >>Makefile.am + printf '%s' 'libndpi_la_SOURCES =' >>Makefile.am + + # add all *.c files found to SOURCES + for src in ${C_SOURCES}; do + printf ' \\\n\t%s' "${src}" >>Makefile.am + done + printf '\n\n' >>Makefile.am + + # libtool CFLAGS / LDFLAGS +cat >>Makefile.am <Makefile.am <>Makefile.am + done + printf '\n\n' >>Makefile.am + + printf '%s\n%s\n' \ + 'fuzz_ndpi_reader_seed_corpus.zip: $(TESTPCAPS)' \ + ' zip -r fuzz_ndpi_reader_seed_corpus.zip $(TESTPCAPS)' >>Makefile.am + + cd "${MYDIR}" +} + +generate_src_am +generate_fuzz_am -- cgit v1.2.3