From e082134a0d369d32e08e1283941883e02c07e6d0 Mon Sep 17 00:00:00 2001 From: Philippe Antoine Date: Fri, 24 Jan 2020 08:14:44 +0100 Subject: Fix sanitizers build on Travis --- .travis.yml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/.travis.yml b/.travis.yml index 57b347449..f18ae0475 100644 --- a/.travis.yml +++ b/.travis.yml @@ -79,14 +79,29 @@ matrix: env: CXXFLAGS="-DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION -fsanitize=address -fsanitize=fuzzer-no-link" CFLAGS="-DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION -fsanitize=address -fsanitize=fuzzer-no-link" LDFLAGS="-fsanitize=address" QA_FUZZ=asan compiler: clang os: linux + addons: + apt: + packages: + - libpcap-dev + - autogen - name: fuzzm env: CXXFLAGS="-DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION -fsanitize=memory -fsanitize=fuzzer-no-link" CFLAGS="-DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION -fsanitize=memory -fsanitize=fuzzer-no-link" LDFLAGS="-fsanitize=memory" QA_FUZZ=msan compiler: clang os: linux + addons: + apt: + packages: + - libpcap-dev + - autogen - name: fuzzu env: CXXFLAGS="-DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION -fsanitize=undefined -fsanitize=fuzzer-no-link" CFLAGS="-DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION -fsanitize=undefined -fno-sanitize-recover=undefined,integer -fsanitize=fuzzer-no-link" LDFLAGS="-fsanitize=undefined" QA_FUZZ=ubsan compiler: clang os: linux + addons: + apt: + packages: + - libpcap-dev + - autogen before_install: -- cgit v1.2.3 From 9fe8bf018ae28304b0ea2ccd3830f4f972876da0 Mon Sep 17 00:00:00 2001 From: Philippe Antoine Date: Fri, 24 Jan 2020 08:24:18 +0100 Subject: Use clang++7 for fuzz targets in Travis --- .travis.yml | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index f18ae0475..84942fa18 100644 --- a/.travis.yml +++ b/.travis.yml @@ -77,31 +77,49 @@ matrix: - name: fuzza env: CXXFLAGS="-DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION -fsanitize=address -fsanitize=fuzzer-no-link" CFLAGS="-DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION -fsanitize=address -fsanitize=fuzzer-no-link" LDFLAGS="-fsanitize=address" QA_FUZZ=asan - compiler: clang os: linux + compiler: clang-7 addons: apt: + sources: + - ubuntu-toolchain-r-test + - llvm-toolchain-trusty-7 packages: + - clang-7 - libpcap-dev - autogen + env: + - MATRIX_EVAL="CC=clang-7 && CXX=clang++-7" - name: fuzzm env: CXXFLAGS="-DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION -fsanitize=memory -fsanitize=fuzzer-no-link" CFLAGS="-DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION -fsanitize=memory -fsanitize=fuzzer-no-link" LDFLAGS="-fsanitize=memory" QA_FUZZ=msan - compiler: clang os: linux + compiler: clang-7 addons: apt: + sources: + - ubuntu-toolchain-r-test + - llvm-toolchain-trusty-7 packages: + - clang-7 - libpcap-dev - autogen + env: + - MATRIX_EVAL="CC=clang-7 && CXX=clang++-7" - name: fuzzu env: CXXFLAGS="-DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION -fsanitize=undefined -fsanitize=fuzzer-no-link" CFLAGS="-DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION -fsanitize=undefined -fno-sanitize-recover=undefined,integer -fsanitize=fuzzer-no-link" LDFLAGS="-fsanitize=undefined" QA_FUZZ=ubsan - compiler: clang os: linux + compiler: clang-7 addons: apt: + sources: + - ubuntu-toolchain-r-test + - llvm-toolchain-trusty-7 packages: + - clang-7 - libpcap-dev - autogen + env: + - MATRIX_EVAL="CC=clang-7 && CXX=clang++-7" before_install: -- cgit v1.2.3 From 6186e1ab01ffcebfe7311e7b57bfdde0d5c4669b Mon Sep 17 00:00:00 2001 From: Philippe Antoine Date: Fri, 24 Jan 2020 08:40:05 +0100 Subject: Only one env --- .travis.yml | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/.travis.yml b/.travis.yml index 84942fa18..0a83dab4d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -76,7 +76,7 @@ matrix: - MATRIX_EVAL="CC=clang-7 && CXX=clang++-7" - name: fuzza - env: CXXFLAGS="-DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION -fsanitize=address -fsanitize=fuzzer-no-link" CFLAGS="-DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION -fsanitize=address -fsanitize=fuzzer-no-link" LDFLAGS="-fsanitize=address" QA_FUZZ=asan + env: CXXFLAGS="-DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION -fsanitize=address -fsanitize=fuzzer-no-link" CFLAGS="-DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION -fsanitize=address -fsanitize=fuzzer-no-link" LDFLAGS="-fsanitize=address" QA_FUZZ=asan CC=clang-7 && CXX=clang++-7 os: linux compiler: clang-7 addons: @@ -88,10 +88,8 @@ matrix: - clang-7 - libpcap-dev - autogen - env: - - MATRIX_EVAL="CC=clang-7 && CXX=clang++-7" - name: fuzzm - env: CXXFLAGS="-DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION -fsanitize=memory -fsanitize=fuzzer-no-link" CFLAGS="-DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION -fsanitize=memory -fsanitize=fuzzer-no-link" LDFLAGS="-fsanitize=memory" QA_FUZZ=msan + env: CXXFLAGS="-DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION -fsanitize=memory -fsanitize=fuzzer-no-link" CFLAGS="-DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION -fsanitize=memory -fsanitize=fuzzer-no-link" LDFLAGS="-fsanitize=memory" QA_FUZZ=msan CC=clang-7 && CXX=clang++-7 os: linux compiler: clang-7 addons: @@ -103,10 +101,8 @@ matrix: - clang-7 - libpcap-dev - autogen - env: - - MATRIX_EVAL="CC=clang-7 && CXX=clang++-7" - name: fuzzu - env: CXXFLAGS="-DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION -fsanitize=undefined -fsanitize=fuzzer-no-link" CFLAGS="-DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION -fsanitize=undefined -fno-sanitize-recover=undefined,integer -fsanitize=fuzzer-no-link" LDFLAGS="-fsanitize=undefined" QA_FUZZ=ubsan + env: CXXFLAGS="-DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION -fsanitize=undefined -fsanitize=fuzzer-no-link" CFLAGS="-DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION -fsanitize=undefined -fno-sanitize-recover=undefined,integer -fsanitize=fuzzer-no-link" LDFLAGS="-fsanitize=undefined" QA_FUZZ=ubsan CC=clang-7 && CXX=clang++-7 os: linux compiler: clang-7 addons: @@ -118,8 +114,6 @@ matrix: - clang-7 - libpcap-dev - autogen - env: - - MATRIX_EVAL="CC=clang-7 && CXX=clang++-7" before_install: @@ -130,6 +124,8 @@ before_script: # - lcov --directory . --zerocounters script: + - echo $QA_FUZZ + - if [ -n "$QA_FUZZ" ]; then echo fuzz ; else echo nofuzz ; fi - if [ -n "$QA_FUZZ" ]; then ./configure --enable-fuzztargets ; else ./configure ; fi - make -- cgit v1.2.3 From 4f230943cf770069651d9f3a9f1cc486f42894c7 Mon Sep 17 00:00:00 2001 From: Philippe Antoine Date: Fri, 24 Jan 2020 09:10:52 +0100 Subject: Debug compilation flag for fuzz targets And adding include file to fuzz_ndpi_reader --- .travis.yml | 8 +++----- fuzz/fuzz_ndpi_reader.c | 1 + 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 0a83dab4d..7c079a985 100644 --- a/.travis.yml +++ b/.travis.yml @@ -76,7 +76,7 @@ matrix: - MATRIX_EVAL="CC=clang-7 && CXX=clang++-7" - name: fuzza - env: CXXFLAGS="-DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION -fsanitize=address -fsanitize=fuzzer-no-link" CFLAGS="-DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION -fsanitize=address -fsanitize=fuzzer-no-link" LDFLAGS="-fsanitize=address" QA_FUZZ=asan CC=clang-7 && CXX=clang++-7 + env: CXXFLAGS="-g -DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION -fsanitize=address -fsanitize=fuzzer-no-link" CFLAGS="-g -DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION -fsanitize=address -fsanitize=fuzzer-no-link" LDFLAGS="-fsanitize=address" QA_FUZZ=asan CC=clang-7 && CXX=clang++-7 os: linux compiler: clang-7 addons: @@ -89,7 +89,7 @@ matrix: - libpcap-dev - autogen - name: fuzzm - env: CXXFLAGS="-DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION -fsanitize=memory -fsanitize=fuzzer-no-link" CFLAGS="-DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION -fsanitize=memory -fsanitize=fuzzer-no-link" LDFLAGS="-fsanitize=memory" QA_FUZZ=msan CC=clang-7 && CXX=clang++-7 + env: CXXFLAGS="-g -DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION -fsanitize=memory -fsanitize=fuzzer-no-link" CFLAGS="-g -DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION -fsanitize=memory -fsanitize=fuzzer-no-link" LDFLAGS="-fsanitize=memory" QA_FUZZ=msan CC=clang-7 && CXX=clang++-7 os: linux compiler: clang-7 addons: @@ -102,7 +102,7 @@ matrix: - libpcap-dev - autogen - name: fuzzu - env: CXXFLAGS="-DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION -fsanitize=undefined -fsanitize=fuzzer-no-link" CFLAGS="-DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION -fsanitize=undefined -fno-sanitize-recover=undefined,integer -fsanitize=fuzzer-no-link" LDFLAGS="-fsanitize=undefined" QA_FUZZ=ubsan CC=clang-7 && CXX=clang++-7 + env: CXXFLAGS="-g -DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION -fsanitize=undefined -fsanitize=fuzzer-no-link" CFLAGS="-g -DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION -fsanitize=undefined -fno-sanitize-recover=undefined,integer -fsanitize=fuzzer-no-link" LDFLAGS="-fsanitize=undefined" QA_FUZZ=ubsan CC=clang-7 && CXX=clang++-7 os: linux compiler: clang-7 addons: @@ -124,8 +124,6 @@ before_script: # - lcov --directory . --zerocounters script: - - echo $QA_FUZZ - - if [ -n "$QA_FUZZ" ]; then echo fuzz ; else echo nofuzz ; fi - if [ -n "$QA_FUZZ" ]; then ./configure --enable-fuzztargets ; else ./configure ; fi - make diff --git a/fuzz/fuzz_ndpi_reader.c b/fuzz/fuzz_ndpi_reader.c index aee2a0b64..1a59d35f0 100644 --- a/fuzz/fuzz_ndpi_reader.c +++ b/fuzz/fuzz_ndpi_reader.c @@ -1,4 +1,5 @@ #include "reader_util.h" +#include "ndpi_api.h" #include -- cgit v1.2.3 From 4749e61bdb96746a216abffaae069bd6ae0edf2f Mon Sep 17 00:00:00 2001 From: Philippe Antoine Date: Fri, 24 Jan 2020 09:23:56 +0100 Subject: Debug symbols in LDFLAGS --- .travis.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 7c079a985..f991f42c3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -76,7 +76,7 @@ matrix: - MATRIX_EVAL="CC=clang-7 && CXX=clang++-7" - name: fuzza - env: CXXFLAGS="-g -DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION -fsanitize=address -fsanitize=fuzzer-no-link" CFLAGS="-g -DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION -fsanitize=address -fsanitize=fuzzer-no-link" LDFLAGS="-fsanitize=address" QA_FUZZ=asan CC=clang-7 && CXX=clang++-7 + env: CXXFLAGS="-g -DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION -fsanitize=address -fsanitize=fuzzer-no-link" CFLAGS="-g -DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION -fsanitize=address -fsanitize=fuzzer-no-link" LDFLAGS="-g -fsanitize=address" QA_FUZZ=asan CC=clang-7 && CXX=clang++-7 os: linux compiler: clang-7 addons: @@ -89,7 +89,7 @@ matrix: - libpcap-dev - autogen - name: fuzzm - env: CXXFLAGS="-g -DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION -fsanitize=memory -fsanitize=fuzzer-no-link" CFLAGS="-g -DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION -fsanitize=memory -fsanitize=fuzzer-no-link" LDFLAGS="-fsanitize=memory" QA_FUZZ=msan CC=clang-7 && CXX=clang++-7 + env: CXXFLAGS="-g -DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION -fsanitize=memory -fsanitize=fuzzer-no-link" CFLAGS="-g -DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION -fsanitize=memory -fsanitize=fuzzer-no-link" LDFLAGS="-g -fsanitize=memory" QA_FUZZ=msan CC=clang-7 && CXX=clang++-7 os: linux compiler: clang-7 addons: @@ -102,7 +102,7 @@ matrix: - libpcap-dev - autogen - name: fuzzu - env: CXXFLAGS="-g -DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION -fsanitize=undefined -fsanitize=fuzzer-no-link" CFLAGS="-g -DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION -fsanitize=undefined -fno-sanitize-recover=undefined,integer -fsanitize=fuzzer-no-link" LDFLAGS="-fsanitize=undefined" QA_FUZZ=ubsan CC=clang-7 && CXX=clang++-7 + env: CXXFLAGS="-g -DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION -fsanitize=undefined -fsanitize=fuzzer-no-link" CFLAGS="-g -DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION -fsanitize=undefined -fno-sanitize-recover=undefined,integer -fsanitize=fuzzer-no-link" LDFLAGS="-g -fsanitize=undefined" QA_FUZZ=ubsan CC=clang-7 && CXX=clang++-7 os: linux compiler: clang-7 addons: -- cgit v1.2.3 From 5b888670842646b6faee744c5f245ab71238f148 Mon Sep 17 00:00:00 2001 From: Philippe Antoine Date: Fri, 24 Jan 2020 09:46:53 +0100 Subject: Looking for symbolizer --- .travis.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index f991f42c3..ad94476fd 100644 --- a/.travis.yml +++ b/.travis.yml @@ -76,7 +76,7 @@ matrix: - MATRIX_EVAL="CC=clang-7 && CXX=clang++-7" - name: fuzza - env: CXXFLAGS="-g -DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION -fsanitize=address -fsanitize=fuzzer-no-link" CFLAGS="-g -DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION -fsanitize=address -fsanitize=fuzzer-no-link" LDFLAGS="-g -fsanitize=address" QA_FUZZ=asan CC=clang-7 && CXX=clang++-7 + env: CXXFLAGS="-g -DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION -fsanitize=address -fsanitize=fuzzer-no-link" CFLAGS="-g -DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION -fsanitize=address -fsanitize=fuzzer-no-link" LDFLAGS="-g -fsanitize=address" QA_FUZZ=asan CC=clang-7 && CXX=clang++-7 ASAN_SYMBOLIZER_PATH=llvm-symbolizer-7 os: linux compiler: clang-7 addons: @@ -124,6 +124,8 @@ before_script: # - lcov --directory . --zerocounters script: + - which llvm-symbolizer + - which llvm-symbolizer-7 - if [ -n "$QA_FUZZ" ]; then ./configure --enable-fuzztargets ; else ./configure ; fi - make -- cgit v1.2.3 From 902a1c7b4c1f95e442303b0935906a737dcdd446 Mon Sep 17 00:00:00 2001 From: Philippe Antoine Date: Fri, 24 Jan 2020 09:53:06 +0100 Subject: Trying with ASAN_SYMBOLIZER_PATH --- .travis.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index ad94476fd..1d22f1b4a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -76,7 +76,7 @@ matrix: - MATRIX_EVAL="CC=clang-7 && CXX=clang++-7" - name: fuzza - env: CXXFLAGS="-g -DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION -fsanitize=address -fsanitize=fuzzer-no-link" CFLAGS="-g -DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION -fsanitize=address -fsanitize=fuzzer-no-link" LDFLAGS="-g -fsanitize=address" QA_FUZZ=asan CC=clang-7 && CXX=clang++-7 ASAN_SYMBOLIZER_PATH=llvm-symbolizer-7 + env: CXXFLAGS="-g -DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION -fsanitize=address -fsanitize=fuzzer-no-link" CFLAGS="-g -DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION -fsanitize=address -fsanitize=fuzzer-no-link" LDFLAGS="-g -fsanitize=address" QA_FUZZ=asan CC=clang-7 && CXX=clang++-7 ASAN_SYMBOLIZER_PATH=llvm-symbolizer os: linux compiler: clang-7 addons: @@ -124,8 +124,6 @@ before_script: # - lcov --directory . --zerocounters script: - - which llvm-symbolizer - - which llvm-symbolizer-7 - if [ -n "$QA_FUZZ" ]; then ./configure --enable-fuzztargets ; else ./configure ; fi - make -- cgit v1.2.3 From 79e1ec9f500fdcce09078e7d791c241c59930e76 Mon Sep 17 00:00:00 2001 From: Philippe Antoine Date: Fri, 24 Jan 2020 10:03:23 +0100 Subject: Complete absolute path for symbolizer --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 1d22f1b4a..d171ee02a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -76,7 +76,7 @@ matrix: - MATRIX_EVAL="CC=clang-7 && CXX=clang++-7" - name: fuzza - env: CXXFLAGS="-g -DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION -fsanitize=address -fsanitize=fuzzer-no-link" CFLAGS="-g -DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION -fsanitize=address -fsanitize=fuzzer-no-link" LDFLAGS="-g -fsanitize=address" QA_FUZZ=asan CC=clang-7 && CXX=clang++-7 ASAN_SYMBOLIZER_PATH=llvm-symbolizer + env: CXXFLAGS="-g -DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION -fsanitize=address -fsanitize=fuzzer-no-link" CFLAGS="-g -DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION -fsanitize=address -fsanitize=fuzzer-no-link" LDFLAGS="-g -fsanitize=address" QA_FUZZ=asan CC=clang-7 && CXX=clang++-7 ASAN_SYMBOLIZER_PATH=/usr/local/clang-7.0.0/bin/llvm-symbolizer os: linux compiler: clang-7 addons: -- cgit v1.2.3 From dc5fecfc9246b304f7404d976986ea8fad7b778d Mon Sep 17 00:00:00 2001 From: Philippe Antoine Date: Fri, 24 Jan 2020 10:15:16 +0100 Subject: Use MSAN_SYMBOLIZER_PATH now that it works --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index d171ee02a..057e93679 100644 --- a/.travis.yml +++ b/.travis.yml @@ -89,7 +89,7 @@ matrix: - libpcap-dev - autogen - name: fuzzm - env: CXXFLAGS="-g -DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION -fsanitize=memory -fsanitize=fuzzer-no-link" CFLAGS="-g -DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION -fsanitize=memory -fsanitize=fuzzer-no-link" LDFLAGS="-g -fsanitize=memory" QA_FUZZ=msan CC=clang-7 && CXX=clang++-7 + env: CXXFLAGS="-g -DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION -fsanitize=memory -fsanitize=fuzzer-no-link" CFLAGS="-g -DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION -fsanitize=memory -fsanitize=fuzzer-no-link" LDFLAGS="-g -fsanitize=memory" QA_FUZZ=msan CC=clang-7 && CXX=clang++-7 MSAN_SYMBOLIZER_PATH=/usr/local/clang-7.0.0/bin/llvm-symbolizer os: linux compiler: clang-7 addons: -- cgit v1.2.3 From e20c4acbde5f277bea70aba3e2be4f45ac382e38 Mon Sep 17 00:00:00 2001 From: Alfredo Cardigliano Date: Fri, 24 Jan 2020 18:36:38 +0100 Subject: Implement ndpi_flowv6_flow_hash ndpi_flowv4_flow_hash. Add ndpi_base64_encode. --- src/include/ndpi_api.h | 1 + src/lib/ndpi_community_id.c | 372 ++++++++++++++++++++++++++++++++++++++++++++ src/lib/ndpi_main.c | 24 --- src/lib/ndpi_utils.c | 49 ++++++ 4 files changed, 422 insertions(+), 24 deletions(-) create mode 100644 src/lib/ndpi_community_id.c diff --git a/src/include/ndpi_api.h b/src/include/ndpi_api.h index 5723494ba..342a24ae7 100644 --- a/src/include/ndpi_api.h +++ b/src/include/ndpi_api.h @@ -857,6 +857,7 @@ extern "C" { void ndpi_user_pwd_payload_copy(u_int8_t *dest, u_int dest_len, u_int offset, const u_int8_t *src, u_int src_len); u_char* ndpi_base64_decode(const u_char *src, size_t len, size_t *out_len); + char* ndpi_base64_encode(unsigned char const* bytes_to_encode, ssize_t in_len); int ndpi_load_ipv4_ptree(struct ndpi_detection_module_struct *ndpi_str, const char *path, u_int16_t protocol_id); diff --git a/src/lib/ndpi_community_id.c b/src/lib/ndpi_community_id.c new file mode 100644 index 000000000..a7602bc24 --- /dev/null +++ b/src/lib/ndpi_community_id.c @@ -0,0 +1,372 @@ +/* + * ndpi_community_id.c + * + * Copyright (C) 2011-20 - ntop.org + * + * This file is part of nDPI, an open source deep packet inspection + * library based on the OpenDPI and PACE technology by ipoque GmbH + * + * nDPI is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * nDPI is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with nDPI. If not, see . + * + */ + +#ifdef HAVE_CONFIG_H +#include "ndpi_config.h" +#endif + +#include +#include +#include + +#include "ndpi_api.h" +#include "ndpi_config.h" + +#include +#ifndef WIN32 +#include +#endif + +#if defined __FreeBSD__ || defined __NetBSD__ || defined __OpenBSD__ +#include +#endif + +#include "ndpi_sha1.h" + +#define NDPI_ICMP6_ECHO_REQUEST 128 +#define NDPI_ICMP6_ECHO_REPLY 129 +#define NDPI_MLD_LISTENER_QUERY 130 +#define NDPI_MLD_LISTENER_REPORT 131 + +#define NDPI_ROUTER_SOLICIT 133 +#define NDPI_ROUTER_ADVERT 134 +#define NDPI_NEIGHBOR_SOLICIT 135 +#define NDPI_NEIGHBOR_ADVERT 136 + +#define NDPI_ICMP_ECHOREPLY 0 +#define NDPI_ICMP_ECHO 8 +#define NDPI_ICMP_ROUTERADVERT 9 +#define NDPI_ICMP_ROUTERSOLICIT 10 +#define NDPI_ICMP_TIMESTAMP 13 +#define NDPI_ICMP_TIMESTAMPREPLY 14 +#define NDPI_ICMP_INFO_REQUEST 15 +#define NDPI_ICMP_INFO_REPLY 16 +#define NDPI_ICMP_MASKREQ 17 +#define NDPI_ICMP_MASKREPLY 18 + +#define NDPI_ICMP6_WRUREQUEST 139 +#define NDPI_ICMP6_WRUREPLY 140 + +/* **************************************************** */ + +static ssize_t ndpi_community_id_buf_copy(u_int8_t * const dst, const void * const src, ssize_t len) { + if(src) + memcpy(dst, src, len); + else + memset(dst, 0, len); + + return len; +} + +/* **************************************************** */ + +/* + https://github.com/corelight/community-id-spec/blob/bda913f617389df07cdaa23606e11bbd318e265c/community-id.py#L56 +*/ +static u_int8_t ndpi_community_id_icmp_type_to_code_v4(u_int8_t icmp_type, u_int8_t icmp_code, int *is_one_way) { + *is_one_way = 0; + + switch(icmp_type) { + case NDPI_ICMP_ECHO: + return NDPI_ICMP_ECHOREPLY; + case NDPI_ICMP_ECHOREPLY: + return NDPI_ICMP_ECHO; + case NDPI_ICMP_TIMESTAMP: + return NDPI_ICMP_TIMESTAMPREPLY; + case NDPI_ICMP_TIMESTAMPREPLY: + return NDPI_ICMP_TIMESTAMP; + case NDPI_ICMP_INFO_REQUEST: + return NDPI_ICMP_INFO_REPLY; + case NDPI_ICMP_INFO_REPLY: + return NDPI_ICMP_INFO_REQUEST; + case NDPI_ICMP_ROUTERSOLICIT: + return NDPI_ICMP_ROUTERADVERT; + case NDPI_ICMP_ROUTERADVERT: + return NDPI_ICMP_ROUTERSOLICIT; + case NDPI_ICMP_MASKREQ: + return NDPI_ICMP_MASKREPLY; + case NDPI_ICMP_MASKREPLY: + return NDPI_ICMP_MASKREQ; + default: + *is_one_way = 1; + return icmp_code; + } +} + +/* **************************************************** */ + +/* + https://github.com/corelight/community-id-spec/blob/bda913f617389df07cdaa23606e11bbd318e265c/community-id.py#L83 +*/ +static u_int8_t ndpi_community_id_icmp_type_to_code_v6(u_int8_t icmp_type, u_int8_t icmp_code, int *is_one_way) { + *is_one_way = 0; + + switch(icmp_type) { + case NDPI_ICMP6_ECHO_REQUEST: + return NDPI_ICMP6_ECHO_REPLY; + case NDPI_ICMP6_ECHO_REPLY: + return NDPI_ICMP6_ECHO_REQUEST; + case NDPI_ROUTER_SOLICIT: + return NDPI_ROUTER_ADVERT; + case NDPI_ROUTER_ADVERT: + return NDPI_ROUTER_SOLICIT; + case NDPI_NEIGHBOR_SOLICIT: + return NDPI_NEIGHBOR_ADVERT; + case NDPI_NEIGHBOR_ADVERT: + return NDPI_NEIGHBOR_SOLICIT; + case NDPI_MLD_LISTENER_QUERY: + return NDPI_MLD_LISTENER_REPORT; + case NDPI_MLD_LISTENER_REPORT: + return NDPI_MLD_LISTENER_QUERY; + case NDPI_ICMP6_WRUREQUEST: + return NDPI_ICMP6_WRUREPLY; + case NDPI_ICMP6_WRUREPLY: + return NDPI_ICMP6_WRUREQUEST; + // Home Agent Address Discovery Request Message and reply + case 144: + return 145; + case 145: + return 144; + default: + *is_one_way = 1; + return icmp_code; + } +} + +/* **************************************************** */ + +/* + https://github.com/corelight/community-id-spec/blob/bda913f617389df07cdaa23606e11bbd318e265c/community-id.py#L164 +*/ +static int ndpi_community_id_peer_v4_is_less_than(u_int32_t ip1, u_int32_t ip2, u_int16_t p1, u_int16_t p2) { + int comp = memcmp(&ip1, &ip2, sizeof(u_int32_t)); + return comp < 0 || (comp == 0 && p1 < p2); +} + +/* **************************************************** */ + +static int ndpi_community_id_peer_v6_is_less_than(struct ndpi_in6_addr *ip1, struct ndpi_in6_addr *ip2, u_int16_t p1, u_int16_t p2) { + int comp = memcmp(ip1, ip2, sizeof(struct ndpi_in6_addr)); + return comp < 0 || (comp == 0 && p1 < p2); +} + +/* **************************************************** */ + +static void ndpi_community_id_sha1_hash(const uint8_t *message, size_t len, u_char *hash /* 20-bytes */) { + SHA1_CTX ctx; + + SHA1Init(&ctx); + SHA1Update(&ctx, message, len); + SHA1Final(hash, &ctx); + + ndpi_free(&ctx); +} + +/* **************************************************** */ + +/* +https://github.com/corelight/community-id-spec/blob/bda913f617389df07cdaa23606e11bbd318e265c/community-id.py#L285 +*/ +static int ndpi_community_id_finalize_and_compute_hash(u_int8_t *comm_buf, u_int8_t l4_proto, + u_int16_t src_port, u_int16_t dst_port, char *hash_buf, u_int8_t hash_buf_len) { + u_int16_t off = 0; + uint32_t hash[5]; + char *community_id; + + /* L4 proto */ + off += ndpi_community_id_buf_copy(&comm_buf[off], &l4_proto, sizeof(l4_proto)); + + /* Pad */ + off += ndpi_community_id_buf_copy(&comm_buf[off], NULL, 1); + + /* Source and destination ports */ + switch(l4_proto) { + case IPPROTO_ICMP: + case IPPROTO_ICMPV6: + case IPPROTO_SCTP: + case IPPROTO_UDP: + case IPPROTO_TCP: + src_port = htons(src_port); + dst_port = htons(dst_port); + off += ndpi_community_id_buf_copy(&comm_buf[off], &src_port, sizeof(src_port)); + off += ndpi_community_id_buf_copy(&comm_buf[off], &dst_port, sizeof(dst_port)); + break; + } + + /* Compute the sha1 of the result */ + ndpi_community_id_sha1_hash(comm_buf, off, (u_char*)hash); + + /* First, we bring everything into network-byte-order */ + for(u_int i = 0; i < 5; i++) + hash[i] = htonl(hash[i]); + + /* Finally the base64 encoding */ + community_id = ndpi_base64_encode((u_int8_t*)hash, sizeof(hash)); + + if (community_id == NULL) + return -1; + +#if 0 /* Debug Info */ + printf("Hex output: "); + for(int i = 0; i < off; i++) + printf("%.2x ", comm_buf[i]); + printf("\n"); + + printf("Sha1 sum: "); + for(int i = 0; i < 5; i++) + printf("%.2x ", ntohl(hash[i])); + printf("\n"); + + printf("Base64: %s\n", community_id); +#endif + + if (hash_buf_len < strlen(community_id)+1) { + ndpi_free(community_id); + return -1; + } + + strcpy(hash_buf, community_id); + ndpi_free(community_id); + + return 0; +} + +/* **************************************************** */ + +/* + NOTE: + - Leave fields empty/zero when information is missing (e.g. with ICMP ports are zero) + - The hash_buf most be 30+1 bits or longer + - Return code: 0 = OK, -1 otherwise +*/ + +int ndpi_flowv4_flow_hash(u_int8_t l4_proto, u_int32_t src_ip, u_int32_t dst_ip, + u_int16_t src_port, u_int16_t dst_port, + u_int8_t icmp_type, u_int8_t icmp_code, + u_char *hash_buf, u_int8_t hash_buf_len) { + /* + Input buffer (40 bytes) + 2 - Seed + 16 - IPv6 src + 16 - IPv6 dst + 1 - L4 proto + 1 - Pad + 2 - Port src + 2 - Port dst + */ + u_int8_t comm_buf[40] = { 0 }; + u_int16_t off = 0; + u_int16_t seed = 0; + u_int32_t *ip_a_ptr, *ip_b_ptr; + u_int16_t port_a, port_b; + int icmp_one_way = 0; + + /* Adjust the ports according to the specs */ + switch(l4_proto) { + case IPPROTO_ICMP: + src_port = icmp_type; + dst_port = ndpi_community_id_icmp_type_to_code_v4(icmp_type, icmp_code, &icmp_one_way); + break; + case IPPROTO_SCTP: + case IPPROTO_UDP: + case IPPROTO_TCP: + /* src/dst port ok */ + break; + default: + src_port = dst_port = 0; + break; + } + + /* + The community id hash doesn't have the definition of client and server, it just sorts IP addresses + and ports to make sure the smaller ip address is the first. This performs this check and + possibly swap client ip and port. + */ + if(icmp_one_way || ndpi_community_id_peer_v4_is_less_than(src_ip, dst_ip, src_port, dst_port)) { + ip_a_ptr = &src_ip, ip_b_ptr = &dst_ip; + port_a = src_port, port_b = dst_port; + } else { + /* swap flow peers */ + ip_a_ptr = &dst_ip, ip_b_ptr = &src_ip; + port_a = dst_port, port_b = src_port; + } + + /* Seed */ + off = ndpi_community_id_buf_copy(&comm_buf[off], &seed, sizeof(seed)); + + /* Source and destination IPs */ + off += ndpi_community_id_buf_copy(&comm_buf[off], ip_a_ptr, sizeof(src_ip)); + off += ndpi_community_id_buf_copy(&comm_buf[off], ip_b_ptr, sizeof(dst_ip)); + + return ndpi_community_id_finalize_and_compute_hash(&comm_buf[off], + l4_proto, port_a, port_b, (char*)hash_buf, hash_buf_len); +} + +/* **************************************************** */ + +int ndpi_flowv6_flow_hash(u_int8_t l4_proto, struct ndpi_in6_addr *src_ip, struct ndpi_in6_addr *dst_ip, + u_int16_t src_port, u_int16_t dst_port, + u_int8_t icmp_type, u_int8_t icmp_code, + u_char *hash_buf, u_int8_t hash_buf_len) { + u_int8_t comm_buf[40] = { 0 }; + u_int16_t off = 0; + u_int16_t seed = 0; + struct ndpi_in6_addr *ip_a_ptr, *ip_b_ptr; + u_int16_t port_a, port_b; + int icmp_one_way = 0; + + switch(l4_proto) { + case IPPROTO_ICMPV6: + src_port = icmp_type; + dst_port = ndpi_community_id_icmp_type_to_code_v6(icmp_type, icmp_code, &icmp_one_way); + break; + case IPPROTO_SCTP: + case IPPROTO_UDP: + case IPPROTO_TCP: + /* src/dst port ok */ + break; + default: + src_port = dst_port = 0; + break; + } + + if(icmp_one_way || ndpi_community_id_peer_v6_is_less_than(src_ip, dst_ip, src_port, dst_port)) { + ip_a_ptr = src_ip, ip_b_ptr = dst_ip; + port_a = src_port, port_b = dst_port; + } else { + ip_a_ptr = dst_ip, ip_b_ptr = src_ip; + port_a = dst_port, port_b = src_port; + } + + /* Seed */ + off = ndpi_community_id_buf_copy(&comm_buf[off], &seed, sizeof(seed)); + + /* Source and destination IPs */ + off += ndpi_community_id_buf_copy(&comm_buf[off], ip_a_ptr, sizeof(struct ndpi_in6_addr)); + off += ndpi_community_id_buf_copy(&comm_buf[off], ip_b_ptr, sizeof(struct ndpi_in6_addr)); + + return ndpi_community_id_finalize_and_compute_hash(&comm_buf[off], + l4_proto, port_a, port_b, (char*)hash_buf, hash_buf_len); +} + +/* **************************************************** */ diff --git a/src/lib/ndpi_main.c b/src/lib/ndpi_main.c index 90c9a81a4..9c406d988 100644 --- a/src/lib/ndpi_main.c +++ b/src/lib/ndpi_main.c @@ -6568,30 +6568,6 @@ void ndpi_lru_add_to_cache(struct ndpi_lru_cache *c, u_int32_t key, u_int16_t va /* ******************************************************************** */ -/* - NOTE: - - Leave fields empty/zero when information is missing (e.g. with ICMP ports are zero) - - The hash_buf most be 30+1 bits or longer - - Return code: 0 = OK, -1 otherwise -*/ - -int ndpi_flowv4_flow_hash(u_int8_t l4_proto, u_int32_t src_ip, - u_int32_t dst_ip, u_int16_t src_port, u_int16_t dst_port, - u_int8_t icmp_type, u_int8_t icmp_code, - u_char *hash_buf, u_int8_t hash_buf_len) { - - return(0); /* OK */ -} - -int ndpi_flowv6_flow_hash(u_int8_t l4_proto, struct ndpi_in6_addr *src_ip, struct ndpi_in6_addr *dst_ip, - u_int16_t src_port, u_int16_t dst_port, u_int8_t icmp_type, u_int8_t icmp_code, - u_char *hash_buf, u_int8_t hash_buf_len) { - - return(0); /* OK */ -} - -/* ******************************************************************** */ - /* This function tells if it's possible to further dissect a given flow 0 - All possible dissection has been completed diff --git a/src/lib/ndpi_utils.c b/src/lib/ndpi_utils.c index 7d9c60104..41447bc7b 100644 --- a/src/lib/ndpi_utils.c +++ b/src/lib/ndpi_utils.c @@ -851,6 +851,55 @@ u_char* ndpi_base64_decode(const u_char *src, size_t len, size_t *out_len) { return out; } +/* ********************************** */ + +char* ndpi_base64_encode(unsigned char const* bytes_to_encode, ssize_t in_len) { + ssize_t len = 0, ret_size; + char *ret; + int i = 0; + unsigned char char_array_3[3]; + unsigned char char_array_4[4]; + + ret_size = ((in_len+2)/3)*4; + + if((ret = (char*)ndpi_malloc(ret_size+1)) == NULL) + return NULL; + + while (in_len--) { + char_array_3[i++] = *(bytes_to_encode++); + if(i == 3) { + char_array_4[0] = (char_array_3[0] & 0xfc) >> 2; + char_array_4[1] = ((char_array_3[0] & 0x03) << 4) + ((char_array_3[1] & 0xf0) >> 4); + char_array_4[2] = ((char_array_3[1] & 0x0f) << 2) + ((char_array_3[2] & 0xc0) >> 6); + char_array_4[3] = char_array_3[2] & 0x3f; + + for(i = 0; i < 4; i++) + ret[len++] = base64_table[char_array_4[i]]; + i = 0; + } + } + + if(i) { + for(int j = i; j < 3; j++) + char_array_3[j] = '\0'; + + char_array_4[0] = (char_array_3[0] & 0xfc) >> 2; + char_array_4[1] = ((char_array_3[0] & 0x03) << 4) + ((char_array_3[1] & 0xf0) >> 4); + char_array_4[2] = ((char_array_3[1] & 0x0f) << 2) + ((char_array_3[2] & 0xc0) >> 6); + char_array_4[3] = char_array_3[2] & 0x3f; + + for(int j = 0; (j < i + 1); j++) + ret[len++] = base64_table[char_array_4[j]]; + + while((i++ < 3)) + ret[len++] = '='; + } + + ret[len++] = '\0'; + + return ret; +} + /* ********************************** */ /* ********************************** */ -- cgit v1.2.3 From 87d228f78e6d6515fe080e818881f2bc5adf044c Mon Sep 17 00:00:00 2001 From: Luca Deri Date: Fri, 24 Jan 2020 19:50:38 +0100 Subject: Fixed heap-buffer-overflow in citrix decoder --- src/lib/protocols/citrix.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/protocols/citrix.c b/src/lib/protocols/citrix.c index a05d748ca..65852da7f 100644 --- a/src/lib/protocols/citrix.c +++ b/src/lib/protocols/citrix.c @@ -51,7 +51,7 @@ static void ndpi_check_citrix(struct ndpi_detection_module_struct *ndpi_struct, ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_CITRIX, NDPI_PROTOCOL_UNKNOWN); } return; - } else if(payload_len > 4) { + } else if(payload_len > 22) { char citrix_header[] = { 0x1a, 0x43, 0x47, 0x50, 0x2f, 0x30, 0x31 }; if((memcmp(packet->payload, citrix_header, sizeof(citrix_header)) == 0) -- cgit v1.2.3 From d86d632fe63114792a1fa150f8290e12bfa8f71d Mon Sep 17 00:00:00 2001 From: Luca Deri Date: Fri, 24 Jan 2020 21:04:27 +0100 Subject: Added memory boundary check in HTTP dissector --- fuzz/fuzz_ndpi_reader.c | 116 +++++++++++++++++++++++------------------------ src/lib/protocols/http.c | 2 +- 2 files changed, 59 insertions(+), 59 deletions(-) diff --git a/fuzz/fuzz_ndpi_reader.c b/fuzz/fuzz_ndpi_reader.c index 1a59d35f0..7de3d45f4 100644 --- a/fuzz/fuzz_ndpi_reader.c +++ b/fuzz/fuzz_ndpi_reader.c @@ -18,74 +18,74 @@ u_int8_t human_readeable_string_len = 5; u_int8_t max_num_udp_dissected_pkts = 16 /* 8 is enough for most protocols, Signal requires more */, max_num_tcp_dissected_pkts = 80 /* due to telnet */; int bufferToFile(const char * name, const uint8_t *Data, size_t Size) { - FILE * fd; - if (remove(name) != 0) { - if (errno != ENOENT) { - printf("failed remove, errno=%d\n", errno); - return -1; - } - } - fd = fopen(name, "wb"); - if (fd == NULL) { - printf("failed open, errno=%d\n", errno); - return -2; - } - if (fwrite (Data, 1, Size, fd) != Size) { - fclose(fd); - return -3; + FILE * fd; + if (remove(name) != 0) { + if (errno != ENOENT) { + printf("failed remove, errno=%d\n", errno); + return -1; } + } + fd = fopen(name, "wb"); + if (fd == NULL) { + printf("failed open, errno=%d\n", errno); + return -2; + } + if (fwrite (Data, 1, Size, fd) != Size) { fclose(fd); - return 0; + return -3; + } + fclose(fd); + return 0; } int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) { - pcap_t * pkts; - const u_char *pkt; - struct pcap_pkthdr *header; - int r; - char errbuf[PCAP_ERRBUF_SIZE]; - NDPI_PROTOCOL_BITMASK all; + pcap_t * pkts; + const u_char *pkt; + struct pcap_pkthdr *header; + int r; + char errbuf[PCAP_ERRBUF_SIZE]; + NDPI_PROTOCOL_BITMASK all; + if (prefs == NULL) { + prefs = calloc(sizeof(struct ndpi_workflow_prefs), 1); if (prefs == NULL) { - prefs = calloc(sizeof(struct ndpi_workflow_prefs), 1); - if (prefs == NULL) { - //should not happen - return 1; - } - prefs->decode_tunnels = 1; - prefs->num_roots = 16; - prefs->max_ndpi_flows = 1024; - prefs->quiet_mode = 0; + //should not happen + return 1; } - bufferToFile("/tmp/fuzz.pcap", Data, Size); + prefs->decode_tunnels = 1; + prefs->num_roots = 16; + prefs->max_ndpi_flows = 1024; + prefs->quiet_mode = 0; + } + bufferToFile("/tmp/fuzz.pcap", Data, Size); - pkts = pcap_open_offline("/tmp/fuzz.pcap", errbuf); - if (pkts == NULL) { - return 0; - } - struct ndpi_workflow * workflow = ndpi_workflow_init(prefs, pkts); - // enable all protocols - NDPI_BITMASK_SET_ALL(all); - ndpi_set_protocol_detection_bitmask2(workflow->ndpi_struct, &all); - memset(workflow->stats.protocol_counter, 0, - sizeof(workflow->stats.protocol_counter)); - memset(workflow->stats.protocol_counter_bytes, 0, - sizeof(workflow->stats.protocol_counter_bytes)); - memset(workflow->stats.protocol_flows, 0, - sizeof(workflow->stats.protocol_flows)); - ndpi_finalize_initalization(workflow->ndpi_struct); + pkts = pcap_open_offline("/tmp/fuzz.pcap", errbuf); + if (pkts == NULL) { + return 0; + } + struct ndpi_workflow * workflow = ndpi_workflow_init(prefs, pkts); + // enable all protocols + NDPI_BITMASK_SET_ALL(all); + ndpi_set_protocol_detection_bitmask2(workflow->ndpi_struct, &all); + memset(workflow->stats.protocol_counter, 0, + sizeof(workflow->stats.protocol_counter)); + memset(workflow->stats.protocol_counter_bytes, 0, + sizeof(workflow->stats.protocol_counter_bytes)); + memset(workflow->stats.protocol_flows, 0, + sizeof(workflow->stats.protocol_flows)); + ndpi_finalize_initalization(workflow->ndpi_struct); + r = pcap_next_ex(pkts, &header, &pkt); + while (r > 0) { + /* allocate an exact size buffer to check overflows */ + uint8_t *packet_checked = malloc(header->caplen); + memcpy(packet_checked, pkt, header->caplen); + ndpi_workflow_process_packet(workflow, header, packet_checked); + free(packet_checked); r = pcap_next_ex(pkts, &header, &pkt); - while (r > 0) { - /* allocate an exact size buffer to check overflows */ - uint8_t *packet_checked = malloc(header->caplen); - memcpy(packet_checked, pkt, header->caplen); - ndpi_workflow_process_packet(workflow, header, packet_checked); - free(packet_checked); - r = pcap_next_ex(pkts, &header, &pkt); - } - ndpi_workflow_free(workflow); - pcap_close(pkts); + } + ndpi_workflow_free(workflow); + pcap_close(pkts); - return 0; + return 0; } diff --git a/src/lib/protocols/http.c b/src/lib/protocols/http.c index 14be88246..00522cfb9 100644 --- a/src/lib/protocols/http.c +++ b/src/lib/protocols/http.c @@ -300,7 +300,7 @@ static void check_content_type_and_change_protocol(struct ndpi_detection_module_ setHttpUserAgent(ndpi_struct, flow, token); } } - } else if(memcmp(ua, "netflix-ios-app", 15) == 0) { + } else if((packet->user_agent_line.len > 14) && (memcmp(ua, "netflix-ios-app", 15) == 0)) { NDPI_LOG_INFO(ndpi_struct, "found netflix\n"); ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_NETFLIX, NDPI_PROTOCOL_CATEGORY_STREAMING); return; -- cgit v1.2.3 From ab2a011cf9b4a6cf4ffb82726609154d6c09b753 Mon Sep 17 00:00:00 2001 From: Alfredo Cardigliano Date: Sat, 25 Jan 2020 16:04:00 +0100 Subject: Community ID hash fixes --- src/lib/ndpi_community_id.c | 44 ++++++++++++++++++++++++-------------------- 1 file changed, 24 insertions(+), 20 deletions(-) diff --git a/src/lib/ndpi_community_id.c b/src/lib/ndpi_community_id.c index a7602bc24..7b55fc009 100644 --- a/src/lib/ndpi_community_id.c +++ b/src/lib/ndpi_community_id.c @@ -173,13 +173,10 @@ static int ndpi_community_id_peer_v6_is_less_than(struct ndpi_in6_addr *ip1, str /* **************************************************** */ static void ndpi_community_id_sha1_hash(const uint8_t *message, size_t len, u_char *hash /* 20-bytes */) { - SHA1_CTX ctx; - + SHA1_CTX ctx = { 0 }; SHA1Init(&ctx); SHA1Update(&ctx, message, len); SHA1Final(hash, &ctx); - - ndpi_free(&ctx); } /* **************************************************** */ @@ -187,9 +184,9 @@ static void ndpi_community_id_sha1_hash(const uint8_t *message, size_t len, u_ch /* https://github.com/corelight/community-id-spec/blob/bda913f617389df07cdaa23606e11bbd318e265c/community-id.py#L285 */ -static int ndpi_community_id_finalize_and_compute_hash(u_int8_t *comm_buf, u_int8_t l4_proto, +static int ndpi_community_id_finalize_and_compute_hash(u_int8_t *comm_buf, u_int16_t off, u_int8_t l4_proto, u_int16_t src_port, u_int16_t dst_port, char *hash_buf, u_int8_t hash_buf_len) { - u_int16_t off = 0; + u_int8_t pad = 0; uint32_t hash[5]; char *community_id; @@ -197,7 +194,7 @@ static int ndpi_community_id_finalize_and_compute_hash(u_int8_t *comm_buf, u_int off += ndpi_community_id_buf_copy(&comm_buf[off], &l4_proto, sizeof(l4_proto)); /* Pad */ - off += ndpi_community_id_buf_copy(&comm_buf[off], NULL, 1); + off += ndpi_community_id_buf_copy(&comm_buf[off], &pad, sizeof(pad)); /* Source and destination ports */ switch(l4_proto) { @@ -206,27 +203,21 @@ static int ndpi_community_id_finalize_and_compute_hash(u_int8_t *comm_buf, u_int case IPPROTO_SCTP: case IPPROTO_UDP: case IPPROTO_TCP: - src_port = htons(src_port); - dst_port = htons(dst_port); off += ndpi_community_id_buf_copy(&comm_buf[off], &src_port, sizeof(src_port)); off += ndpi_community_id_buf_copy(&comm_buf[off], &dst_port, sizeof(dst_port)); break; } - /* Compute the sha1 of the result */ + /* Compute SHA1 */ ndpi_community_id_sha1_hash(comm_buf, off, (u_char*)hash); - /* First, we bring everything into network-byte-order */ - for(u_int i = 0; i < 5; i++) - hash[i] = htonl(hash[i]); - - /* Finally the base64 encoding */ + /* Base64 encoding */ community_id = ndpi_base64_encode((u_int8_t*)hash, sizeof(hash)); if (community_id == NULL) return -1; -#if 0 /* Debug Info */ +#if 1 /* Debug Info */ printf("Hex output: "); for(int i = 0; i < off; i++) printf("%.2x ", comm_buf[i]); @@ -240,12 +231,15 @@ static int ndpi_community_id_finalize_and_compute_hash(u_int8_t *comm_buf, u_int printf("Base64: %s\n", community_id); #endif - if (hash_buf_len < strlen(community_id)+1) { + if (hash_buf_len < 2 || hash_buf_len-2 < strlen(community_id)+1) { ndpi_free(community_id); return -1; } - strcpy(hash_buf, community_id); + /* Writing hash */ + hash_buf[0] = '1'; + hash_buf[1] = ':'; + strcpy(&hash_buf[2], community_id); ndpi_free(community_id); return 0; @@ -297,6 +291,12 @@ int ndpi_flowv4_flow_hash(u_int8_t l4_proto, u_int32_t src_ip, u_int32_t dst_ip, break; } + /* Convert tuple to NBO */ + src_ip = htonl(src_ip); + dst_ip = htonl(dst_ip); + src_port = htons(src_port); + dst_port = htons(dst_port); + /* The community id hash doesn't have the definition of client and server, it just sorts IP addresses and ports to make sure the smaller ip address is the first. This performs this check and @@ -318,7 +318,7 @@ int ndpi_flowv4_flow_hash(u_int8_t l4_proto, u_int32_t src_ip, u_int32_t dst_ip, off += ndpi_community_id_buf_copy(&comm_buf[off], ip_a_ptr, sizeof(src_ip)); off += ndpi_community_id_buf_copy(&comm_buf[off], ip_b_ptr, sizeof(dst_ip)); - return ndpi_community_id_finalize_and_compute_hash(&comm_buf[off], + return ndpi_community_id_finalize_and_compute_hash(comm_buf, off, l4_proto, port_a, port_b, (char*)hash_buf, hash_buf_len); } @@ -350,6 +350,10 @@ int ndpi_flowv6_flow_hash(u_int8_t l4_proto, struct ndpi_in6_addr *src_ip, struc break; } + /* Convert tuple to NBO */ + src_port = htons(src_port); + dst_port = htons(dst_port); + if(icmp_one_way || ndpi_community_id_peer_v6_is_less_than(src_ip, dst_ip, src_port, dst_port)) { ip_a_ptr = src_ip, ip_b_ptr = dst_ip; port_a = src_port, port_b = dst_port; @@ -365,7 +369,7 @@ int ndpi_flowv6_flow_hash(u_int8_t l4_proto, struct ndpi_in6_addr *src_ip, struc off += ndpi_community_id_buf_copy(&comm_buf[off], ip_a_ptr, sizeof(struct ndpi_in6_addr)); off += ndpi_community_id_buf_copy(&comm_buf[off], ip_b_ptr, sizeof(struct ndpi_in6_addr)); - return ndpi_community_id_finalize_and_compute_hash(&comm_buf[off], + return ndpi_community_id_finalize_and_compute_hash(comm_buf, off, l4_proto, port_a, port_b, (char*)hash_buf, hash_buf_len); } -- cgit v1.2.3 From c84ed5a71257d1db28036c37e90f828ed86623a0 Mon Sep 17 00:00:00 2001 From: Luca Deri Date: Wed, 29 Jan 2020 00:41:35 +0100 Subject: Added check for libpcre presence --- configure.seed | 2 ++ 1 file changed, 2 insertions(+) diff --git a/configure.seed b/configure.seed index 326242cd2..68997975e 100644 --- a/configure.seed +++ b/configure.seed @@ -160,6 +160,8 @@ AM_CONDITIONAL([HAS_FUZZLDFLAGS], [test "x$has_sanitizefuzzer" = "xyes"]) AC_CHECK_LIB(pthread, pthread_setaffinity_np, AC_DEFINE_UNQUOTED(HAVE_PTHREAD_SETAFFINITY_NP, 1, [libc has pthread_setaffinity_np])) +AC_CHECK_LIB(pcre, pcre_compile, AC_DEFINE_UNQUOTED(HAVE_PCRE, 1, [libpcre(-dev) is present])) + AC_CONFIG_FILES([Makefile example/Makefile example/Makefile.dpdk tests/Makefile libndpi.pc src/include/ndpi_define.h src/lib/Makefile python/Makefile fuzz/Makefile]) AC_CONFIG_HEADERS(src/include/ndpi_config.h) AC_SUBST(GIT_RELEASE) -- cgit v1.2.3