diff options
| -rw-r--r-- | .travis.yml | 3 | ||||
| -rw-r--r-- | dependencies/nDPIsrvd.py | 76 | ||||
| -rwxr-xr-x | examples/py-flow-info/flow-info.py | 2 | ||||
| -rwxr-xr-x | examples/py-flow-undetected-to-pcap/flow-undetected-to-pcap.py | 2 | ||||
| -rwxr-xr-x | examples/py-json-stdout/json-stdout.py | 2 | ||||
| -rwxr-xr-x | examples/py-risky-flow-to-pcap/risky-flow-to-pcap.py | 2 | ||||
| -rwxr-xr-x | examples/py-schema-validation/py-schema-validation.py | 7 | ||||
| -rw-r--r-- | schema/daemon_event_schema.json | 3 | ||||
| -rwxr-xr-x | test/run_tests.sh | 93 |
9 files changed, 155 insertions, 35 deletions
diff --git a/.travis.yml b/.travis.yml index 7a14474e0..b41e94d0a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,7 @@ language: c before_install: - sudo apt-get -qq update -- sudo apt-get install -y build-essential make binutils gcc autoconf automake libtool pkg-config git libpcap-dev libgcrypt-dev libgpg-error-dev libjson-c-dev libmaxminddb-dev netcat-openbsd +- sudo apt-get install -y build-essential make binutils gcc autoconf automake libtool pkg-config git libpcap-dev libgcrypt-dev libgpg-error-dev libjson-c-dev libmaxminddb-dev netcat-openbsd python3 python3-jsonschema script: - git clone https://github.com/ntop/nDPI.git - cd nDPI && @@ -10,6 +10,7 @@ script: make install -j4 && cd .. - make -f Makefile.old NDPI_WITH_GCRYPT=yes CUSTOM_LIBNDPI=./nDPI/_install/lib/libndpi.a ENABLE_DEBUG=yes ENABLE_SANITIZER=yes all examples +- env python3 --version - test/run_tests.sh nDPI - make -f Makefile.old clean - PKG_CONFIG_PATH="$(realpath ./nDPI/_install/lib/pkgconfig)" make -f Makefile.old PKG_CONFIG_BIN=pkg-config ENABLE_DEBUG=yes ENABLE_SANITIZER=yes all examples diff --git a/dependencies/nDPIsrvd.py b/dependencies/nDPIsrvd.py index 056557792..d8cccc266 100644 --- a/dependencies/nDPIsrvd.py +++ b/dependencies/nDPIsrvd.py @@ -6,17 +6,22 @@ import base64 import json import re import os -import scapy.all import stat import socket import sys + try: from colorama import Back, Fore, Style USE_COLORAMA=True -except ModuleNotFoundError: - print('Python module colorama not found, using fallback.') +except ImportError: + sys.stderr.write('Python module colorama not found, using fallback.\n') USE_COLORAMA=False +try: + import scapy.all +except ImportError: + sys.stderr.write('Python module scapy not found, PCAP generation will fail!\n') + DEFAULT_HOST = '127.0.0.1' DEFAULT_PORT = 7000 DEFAULT_UNIX = '/tmp/ndpid-distributor.sock' @@ -108,6 +113,52 @@ class FlowManager: return flow +class nDPIsrvdException(Exception): + UNSUPPORTED_ADDRESS_TYPE = 1 + BUFFER_CAPACITY_REACHED = 2 + SOCKET_CONNECTION_BROKEN = 3 + INVALID_LINE_RECEIVED = 4 + CALLBACK_RETURNED_FALSE = 5 + + def __init__(self, etype): + self.etype = etype + def __str__(self): + return 'nDPIsrvdException type {}'.format(self.etype) + +class UnsupportedAddressType(nDPIsrvdException): + def __init__(self, addr): + super().__init__(nDPIsrvdException.UNSUPPORTED_ADDRESS_TYPE) + self.addr = addr + def __str__(self): + return '{}'.format(str(self.addr)) + +class BufferCapacityReached(nDPIsrvdException): + def __init__(self, current_length, max_length): + super().__init__(nDPIsrvdException.BUFFER_CAPACITY_REACHED) + self.current_length = current_length + self.max_length = max_length + def __str__(self): + return '{} of {} bytes'.format(self.current_length, self.max_length) + +class SocketConnectionBroken(nDPIsrvdException): + def __init__(self): + super().__init__(nDPIsrvdException.SOCKET_CONNECTION_BROKEN) + def __str__(self): + return 'Disconnected.' + +class InvalidLineReceived(nDPIsrvdException): + def __init__(self, packet_buffer): + super().__init__(nDPIsrvdException.INVALID_LINE_RECEIVED) + self.packet_buffer = packet_buffer + def __str__(self): + return 'Received JSON line is invalid.' + +class CallbackReturnedFalse(nDPIsrvdException): + def __init__(self): + super().__init__(nDPIsrvdException.CALLBACK_RETURNED_FALSE) + def __str__(self): + return 'Callback returned False, abort.' + class nDPIsrvdSocket: def __init__(self): self.sock_family = None @@ -119,7 +170,7 @@ class nDPIsrvdSocket: elif type(addr) is str: self.sock_family = socket.AF_UNIX else: - raise RuntimeError('Unsupported address type:: {}'.format(str(addr))) + raise UnsupportedAddressType(addr) self.sock = socket.socket(self.sock_family, socket.SOCK_STREAM) self.sock.connect(addr) @@ -130,12 +181,15 @@ class nDPIsrvdSocket: def receive(self): if len(self.buffer) == NETWORK_BUFFER_MAX_SIZE: - raise RuntimeError('Buffer capacity reached ({} bytes), check if it is in sync with nDPId\'s NETWORK_BUFFER_MAX_SIZE.'.format(NETWORK_BUFFER_MAX_SIZE)) + raise BufferCapacityReached(len(self.buffer), NETWORK_BUFFER_MAX_SIZE) - recvd = self.sock.recv(NETWORK_BUFFER_MAX_SIZE - len(self.buffer)) + try: + recvd = self.sock.recv(NETWORK_BUFFER_MAX_SIZE - len(self.buffer)) + except ConnectionResetError: + raise SocketConnectionBroken() if len(recvd) == 0: - raise RuntimeError('Socket connection broken.') + raise SocketConnectionBroken() self.buffer += recvd new_data_avail = False @@ -146,9 +200,9 @@ class nDPIsrvdSocket: if starts_with_digits is None: if len(self.buffer) < NETWORK_BUFFER_MIN_SIZE: break - raise RuntimeError('Invalid packet received: {}'.format(self.buffer)) - self.msglen = int(starts_with_digits[1]) - self.digitlen = len(starts_with_digits[1]) + raise InvalidLineReceived(self.buffer) + self.msglen = int(starts_with_digits.group(1)) + self.digitlen = len(starts_with_digits.group(1)) if len(self.buffer) >= self.msglen + self.digitlen: recvd = self.buffer[self.digitlen:self.msglen + self.digitlen] @@ -179,7 +233,7 @@ class nDPIsrvdSocket: while True: if self.receive() > 0: if self.parse(callback, global_user_data) is False: - raise RuntimeError('Callback returned False, abort.') + raise CallbackReturnedFalse() break; class PcapPacket: diff --git a/examples/py-flow-info/flow-info.py b/examples/py-flow-info/flow-info.py index 3b11a03b7..6f90ec7fa 100755 --- a/examples/py-flow-info/flow-info.py +++ b/examples/py-flow-info/flow-info.py @@ -7,7 +7,7 @@ sys.path.append(os.path.dirname(sys.argv[0]) + '/../usr/share/nDPId') try: import nDPIsrvd from nDPIsrvd import nDPIsrvdSocket, TermColor -except ModuleNotFoundError: +except ImportError: sys.path.append(os.path.dirname(sys.argv[0]) + '/../../dependencies') import nDPIsrvd from nDPIsrvd import nDPIsrvdSocket, TermColor diff --git a/examples/py-flow-undetected-to-pcap/flow-undetected-to-pcap.py b/examples/py-flow-undetected-to-pcap/flow-undetected-to-pcap.py index c25a7601f..e5636312f 100755 --- a/examples/py-flow-undetected-to-pcap/flow-undetected-to-pcap.py +++ b/examples/py-flow-undetected-to-pcap/flow-undetected-to-pcap.py @@ -7,7 +7,7 @@ sys.path.append(os.path.dirname(sys.argv[0]) + '/../usr/share/nDPId') try: import nDPIsrvd from nDPIsrvd import nDPIsrvdSocket, TermColor -except ModuleNotFoundError: +except ImportError: sys.path.append(os.path.dirname(sys.argv[0]) + '/../../dependencies') import nDPIsrvd from nDPIsrvd import nDPIsrvdSocket, TermColor diff --git a/examples/py-json-stdout/json-stdout.py b/examples/py-json-stdout/json-stdout.py index a14447745..8200bf12b 100755 --- a/examples/py-json-stdout/json-stdout.py +++ b/examples/py-json-stdout/json-stdout.py @@ -7,7 +7,7 @@ sys.path.append(os.path.dirname(sys.argv[0]) + '/../usr/share/nDPId') try: import nDPIsrvd from nDPIsrvd import nDPIsrvdSocket, TermColor -except ModuleNotFoundError: +except ImportError: sys.path.append(os.path.dirname(sys.argv[0]) + '/../../dependencies') import nDPIsrvd from nDPIsrvd import nDPIsrvdSocket, TermColor diff --git a/examples/py-risky-flow-to-pcap/risky-flow-to-pcap.py b/examples/py-risky-flow-to-pcap/risky-flow-to-pcap.py index 11165ed35..42004eebe 100755 --- a/examples/py-risky-flow-to-pcap/risky-flow-to-pcap.py +++ b/examples/py-risky-flow-to-pcap/risky-flow-to-pcap.py @@ -8,7 +8,7 @@ sys.path.append(os.path.dirname(sys.argv[0]) + '/../usr/share/nDPId') try: import nDPIsrvd from nDPIsrvd import nDPIsrvdSocket, TermColor -except ModuleNotFoundError: +except ImportError: sys.path.append(os.path.dirname(sys.argv[0]) + '/../../dependencies') import nDPIsrvd from nDPIsrvd import nDPIsrvdSocket, TermColor diff --git a/examples/py-schema-validation/py-schema-validation.py b/examples/py-schema-validation/py-schema-validation.py index 583612516..e8fc390f1 100755 --- a/examples/py-schema-validation/py-schema-validation.py +++ b/examples/py-schema-validation/py-schema-validation.py @@ -7,7 +7,7 @@ sys.path.append(os.path.dirname(sys.argv[0]) + '/../usr/share/nDPId') try: import nDPIsrvd from nDPIsrvd import nDPIsrvdSocket, TermColor -except ModuleNotFoundError: +except ImportError: sys.path.append(os.path.dirname(sys.argv[0]) + '/../../dependencies') import nDPIsrvd from nDPIsrvd import nDPIsrvdSocket, TermColor @@ -43,4 +43,7 @@ if __name__ == '__main__': nsock = nDPIsrvdSocket() nsock.connect(address) - nsock.loop(onJsonLineRecvd, Stats()) + try: + nsock.loop(onJsonLineRecvd, Stats()) + except nDPIsrvd.SocketConnectionBroken as err: + sys.stderr.write('\n{}\n'.format(err)) diff --git a/schema/daemon_event_schema.json b/schema/daemon_event_schema.json index 7a40a6ca0..cf3756236 100644 --- a/schema/daemon_event_schema.json +++ b/schema/daemon_event_schema.json @@ -59,6 +59,9 @@ "tcp-max-post-end-flow-time": { "type": "number" }, + "max-packets-per-flow-to-process": { + "type": "number" + }, "max-packets-per-flow-to-send": { "type": "number" } diff --git a/test/run_tests.sh b/test/run_tests.sh index 747880c9b..4b2603eff 100755 --- a/test/run_tests.sh +++ b/test/run_tests.sh @@ -5,15 +5,6 @@ set -e LINE_SPACES=${LINE_SPACES:-48} MYDIR="$(realpath "$(dirname ${0})")" nDPId_test_EXEC="${2:-"$(realpath "${MYDIR}/../nDPId-test")"}" -nDPI_SOURCE_ROOT="$(realpath "${1}")" -LOCKFILE="$(realpath "${0}").lock" - -touch "${LOCKFILE}" -exec 42< "${LOCKFILE}" -flock -x -n 42 || { - printf '%s\n' "Could not aquire file lock for ${0}. Already running instance?"; - exit 1; -} if [ $# -ne 1 -a $# -ne 2 ]; then cat <<EOF @@ -21,14 +12,30 @@ usage: ${0} [path-to-nDPI-source-root] [path-to-nDPId-test-exec] path-to-nDPId-test-exec defaults to ${nDPId_test_EXEC} EOF -exit 1 +exit 2 fi +nDPI_SOURCE_ROOT="$(realpath "${1}")" +LOCKFILE="$(realpath "${0}").lock" + +touch "${LOCKFILE}" +exec 42< "${LOCKFILE}" +flock -x -n 42 || { + printf '%s\n' "Could not aquire file lock for ${0}. Already running instance?"; + exit 3; +} +function sighandler() +{ + rm -f "${LOCKFILE}" + exit 4 +} +trap sighandler SIGINT SIGTERM + if [ ! -x "${nDPId_test_EXEC}" ]; then cat <<EOF Required nDPId-test executable does not exist; ${nDPId_test_EXEC} EOF -exit 1 +exit 5 fi nDPI_TEST_DIR="${nDPI_SOURCE_ROOT}/tests/pcap" @@ -37,12 +44,17 @@ cat <<EOF nDPId-test......: ${nDPId_test_EXEC} nDPI source root: ${nDPI_TEST_DIR} +-------------------------- +-- nDPId PCAP diff tests -- +-------------------------- + EOF cd "${nDPI_TEST_DIR}" mkdir -p /tmp/nDPId-test-stderr set +e -RETVAL=0 +TESTS_FAILED=0 + for pcap_file in $(ls *.pcap*); do printf '%s\n' "${nDPId_test_EXEC} ${pcap_file}" \ >"/tmp/nDPId-test-stderr/${pcap_file}.out" @@ -56,7 +68,9 @@ for pcap_file in $(ls *.pcap*); do if [ $? -eq 0 ]; then if [ ! -r "${MYDIR}/results/${pcap_file}.out" ]; then printf '%s\n' '[NEW]' - RETVAL=1 + mv -v "${MYDIR}/results/${pcap_file}.out.new" \ + "${MYDIR}/results/${pcap_file}.out" + TESTS_FAILED=$((TESTS_FAILED + 1)) elif diff -u0 "${MYDIR}/results/${pcap_file}.out" \ "${MYDIR}/results/${pcap_file}.out.new" >/dev/null; then printf '%s\n' '[OK]' @@ -66,26 +80,71 @@ for pcap_file in $(ls *.pcap*); do "${MYDIR}/results/${pcap_file}.out.new" mv -v "${MYDIR}/results/${pcap_file}.out.new" \ "${MYDIR}/results/${pcap_file}.out" - RETVAL=1 + TESTS_FAILED=$((TESTS_FAILED + 1)) fi else printf '%s\n' '[FAIL]' printf '%s\n' '----------------------------------------' printf '%s\n' "-- STDERR of ${pcap_file}" cat "/tmp/nDPId-test-stderr/${pcap_file}.out" - RETVAL=1 + TESTS_FAILED=$((TESTS_FAILED + 1)) fi rm -f "${MYDIR}/results/${pcap_file}.out.new" done +cat <<EOF + +---------------------------- +-- JSON schema validation -- +---------------------------- + +EOF + cd "${MYDIR}" for out_file in $(ls results/*.out); do pcap_file="${nDPI_TEST_DIR}/$(basename ${out_file%.out})" if [ ! -r "${pcap_file}" ]; then printf "%-${LINE_SPACES}s\t%s\n" "$(basename ${pcap_file})" '[MISSING]' - RETVAL=1 + TESTS_FAILED=$((TESTS_FAILED + 1)) + else + printf "SCHEMA %-${LINE_SPACES}s\t" "$(basename ${pcap_file})" + printf '%s\n' '*** JSON schema validation ***' >>"/tmp/nDPId-test-stderr/$(basename ${pcap_file}).out" + if [ ! -r "${out_file}" ]; then + printf ' %s\n' '[MISSING]' + TESTS_FAILED=$((TESTS_FAILED + 1)) + continue + fi + cat "${out_file}" | nc -q 1 -l 127.0.0.1 9000 & + nc_pid=$! + ${MYDIR}/../examples/py-schema-validation/py-schema-validation.py \ + --host 127.0.0.1 --port 9000 2>>"/tmp/nDPId-test-stderr/$(basename ${pcap_file}).out" + if [ $? -eq 0 ]; then + printf ' %s\n' '[OK]' + else + printf ' %s\n' '[FAIL]' + printf '%s\n' '----------------------------------------' + printf '%s\n' "-- STDERR of $(basename ${pcap_file})" + cat "/tmp/nDPId-test-stderr/$(basename ${pcap_file}).out" + TESTS_FAILED=$((TESTS_FAILED + 1)) + fi + kill -SIGTERM ${nc_pid} 2>/dev/null + wait ${nc_pid} 2>/dev/null fi done -exit ${RETVAL} +if [ ${TESTS_FAILED} -eq 0 ]; then +cat <<EOF + +-------------------------- +-- All tests succeeded. -- +-------------------------- +EOF + exit 0 +else +cat <<EOF + +*** ${TESTS_FAILED} tests failed. *** +EOF + exit 1 +fi |