Support word diff for tests/do.sh for better readability. (#1565)

* Sync unit tests

Signed-off-by: lns <matzeton@googlemail.com>

3 files changed, 15 insertions, 7 deletions

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index ae50d2069..b94cdb066 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -23,6 +23,7 @@ jobs:
           sudo apt-get install autoconf automake libtool pkg-config gettext libjson-c-dev flex bison libpcap-dev
           sudo apt-get install rrdtool librrd-dev
           sudo apt-get install libpcre3-dev libmaxminddb-dev lcov
+          sudo apt-get install wdiff colordiff
       - name: Configure
         run: env CC=gcc CFLAGS='-Werror' ./autogen.sh --enable-option-checking=fatal --enable-debug-messages --enable-code-coverage --with-pcre --with-maxminddb --enable-tls-sigs
       - name: Build
@@ -288,7 +289,7 @@ jobs:
         if: startsWith(matrix.os, 'macOS') && startsWith(matrix.arch, 'x86_64')
         run: |
           brew install autoconf automake libtool pkg-config gettext json-c
-          brew install coreutils
+          brew install coreutils wdiff colordiff
           brew install rrdtool
       - name: Install MacOS Prerequisites (libgcrypt)
         if: startsWith(matrix.os, 'macOS') && startsWith(matrix.arch, 'x86_64') && startsWith(matrix.gcrypt, '--with-local-libgcrypt')
@@ -389,7 +390,7 @@ jobs:
           args: >
             bash -c
             "apt-get -y update &&
-            apt-get -y install git autoconf automake libtool pkg-config gettext libjson-c-dev flex bison libpcap-dev libgcrypt20-dev libpcre3-dev libmaxminddb-dev rrdtool librrd-dev &&
+            apt-get -y install git wdiff colordiff autoconf automake libtool pkg-config gettext libjson-c-dev flex bison libpcap-dev libgcrypt20-dev libpcre3-dev libmaxminddb-dev rrdtool librrd-dev &&
             git config --global --add safe.directory $(realpath .) &&
             env CC=gcc CFLAGS='-Werror' ./autogen.sh --enable-option-checking=fatal --enable-debug-messages ${{ matrix.gcrypt }} ${{ matrix.msan }} ${{ matrix.pcre }} ${{ matrix.maxminddb }} --enable-tls-sigs &&
             make all &&
@@ -414,7 +415,7 @@ jobs:
           args: >
             bash -c
             "apt-get -y update &&
-            apt-get -y install git autoconf automake libtool pkg-config gettext libjson-c-dev flex bison libpcap-dev libgcrypt20-dev libpcre3-dev libmaxminddb-dev rrdtool librrd-dev &&
+            apt-get -y install git wdiff colordiff autoconf automake libtool pkg-config gettext libjson-c-dev flex bison libpcap-dev libgcrypt20-dev libpcre3-dev libmaxminddb-dev rrdtool librrd-dev &&
             git config --global --add safe.directory $(realpath .) &&
             env CC=gcc CFLAGS='-Werror' ./autogen.sh --enable-option-checking=fatal --enable-debug-messages ${{ matrix.gcrypt }} ${{ matrix.msan }} ${{ matrix.pcre }} ${{ matrix.maxminddb }} --enable-tls-sigs &&
             make all &&
@@ -439,7 +440,7 @@ jobs:
           args: >
             bash -c
             "apt-get -y update &&
-            apt-get -y install git autoconf automake libtool pkg-config gettext libjson-c-dev flex bison libpcap-dev libgcrypt20-dev libpcre3-dev libmaxminddb-dev rrdtool librrd-dev &&
+            apt-get -y install git wdiff colordiff autoconf automake libtool pkg-config gettext libjson-c-dev flex bison libpcap-dev libgcrypt20-dev libpcre3-dev libmaxminddb-dev rrdtool librrd-dev &&
             git config --global --add safe.directory $(realpath .) &&
             env CC=gcc CFLAGS='-Werror' ./autogen.sh --enable-option-checking=fatal --enable-debug-messages ${{ matrix.gcrypt }} ${{ matrix.msan }} ${{ matrix.pcre }} ${{ matrix.maxminddb }} --enable-tls-sigs &&
             make all &&
diff --git a/tests/do.sh.in b/tests/do.sh.in
index 65716c057..99917f5ae 100755
--- a/tests/do.sh.in
+++ b/tests/do.sh.in
@@ -15,6 +15,10 @@ elif [ "${NDPI_TESTS_VALGRIND}" = "1" ]; then
    CMD_PREFIX="valgrind -q --leak-check=full"
 fi
 
+CMD_DIFF="$(which diff)"
+CMD_WDIFF="$(which wdiff)"
+CMD_COLORDIFF="$(which colordiff)"
+
 EXE_SUFFIX=@EXE_SUFFIX@
 PCRE_ENABLED=@PCRE_ENABLED@
 PCRE_PCAPS="WebattackRCE.pcap"
@@ -73,7 +77,7 @@ check_results() {
 	    $CMD
 	    CMD_RET=$?
 	    if [ $CMD_RET -eq 0 ]; then
-	      NUM_DIFF=`diff result/$f.out /tmp/reader.out | wc -l`
+	      NUM_DIFF=`${CMD_DIFF} result/$f.out /tmp/reader.out | wc -l`
 	    else
 	      printf "%-48s\tERROR (ndpiReader${EXE_SUFFIX} exit code: ${CMD_RET})\n" "$f"
 	      RC=$(( RC + 1 ))
@@ -85,7 +89,10 @@ check_results() {
 		else
 		  printf "%-48s\tERROR\n" "$f"
 		  echo "$CMD [old vs new]"
-		  diff result/$f.out /tmp/reader.out
+		  ${CMD_DIFF} result/$f.out /tmp/reader.out
+		  if [ ! -z "${CMD_COLORDIFF}" -a ! -z "${CMD_WDIFF}" ]; then
+		    ${CMD_WDIFF} -n -3 result/$f.out /tmp/reader.out | ${CMD_COLORDIFF}
+		  fi
 		  RC=$(( RC + 1 ))
 	    fi
 
diff --git a/tests/result/mpeg-dash.pcap.out b/tests/result/mpeg-dash.pcap.out
index 919d6638a..0dad1633b 100644
--- a/tests/result/mpeg-dash.pcap.out
+++ b/tests/result/mpeg-dash.pcap.out
@@ -6,7 +6,7 @@ Confidence DPI              : 4 (flows)
 AmazonAWS	9	2693	3
 MpegDash	4	1976	1
 
-	1	TCP 10.84.1.81:60926 <-> 166.248.152.10:80 [proto: 7.291/HTTP.MpegDash][ClearText][Confidence: DPI][cat: Media/1][2 pkts/456 bytes <-> 2 pkts/1520 bytes][Goodput ratio: 72/92][0.30 sec][Hostname/SNI: gdl.news-cdn.site][URL: gdl.news-cdn.site/as/bigo-ad-creatives/3s3/2lOTA7.mp4][StatusCode: 200][User-Agent: Mozilla/5.0 (Linux; Android 11; SM-A715F Build/RP1A.200720.012; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/89.0.4389.105 Mobile Safari/537.36][Risk: ** Suspicious DGA Domain name **][Risk Score: 100][PLAIN TEXT (GET /as/bigo)][Plen Bins: 0,0,0,0,0,0,0,0,0,0,50,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,50,0,0,0,0]
+	1	TCP 10.84.1.81:60926 <-> 166.248.152.10:80 [proto: 7.291/HTTP.MpegDash][ClearText][Confidence: DPI][cat: Media/1][2 pkts/456 bytes <-> 2 pkts/1520 bytes][Goodput ratio: 72/92][0.30 sec][Hostname/SNI: gdl.news-cdn.site][URL: gdl.news-cdn.site/as/bigo-ad-creatives/3s3/2lOTA7.mp4][StatusCode: 200][User-Agent: Mozilla/5.0 (Linux; Android 11; SM-A715F Build/RP1A.200720.012; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/89.0.4389.105 Mobile Safari/537.36][Risk: ** Suspicious DGA Domain name **][Risk Score: 100][Risk Info: gdl.news-cdn.site][PLAIN TEXT (GET /as/bigo)][Plen Bins: 0,0,0,0,0,0,0,0,0,0,50,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,50,0,0,0,0]
 	2	TCP 54.161.101.85:80 <-> 192.168.2.105:59144 [proto: 291.265/MpegDash.AmazonAWS][ClearText][Confidence: DPI][cat: Media/1][2 pkts/1649 bytes <-> 2 pkts/323 bytes][Goodput ratio: 92/59][0.01 sec][Risk: ** Known Proto on Non Std Port **][Risk Score: 50][PLAIN TEXT (OHTTP/1.1 200 OK)][Plen Bins: 0,0,33,0,0,33,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,33,0,0]
 	3	TCP 192.168.2.105:59142 <-> 54.161.101.85:80 [proto: 291.265/MpegDash.AmazonAWS][ClearText][Confidence: DPI][cat: Cloud/13][3 pkts/390 bytes <-> 1 pkts/74 bytes][Goodput ratio: 47/0][0.10 sec][Hostname/SNI: livesim.dashif.org][User-Agent: VLC/3.0.16 LibVLC/3.0.16][Risk: ** Known Proto on Non Std Port **][Risk Score: 50][PLAIN TEXT (IGET /livesim/sts)][Plen Bins: 0,0,0,0,0,100,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
 	4	TCP 192.168.2.105:59146 -> 54.161.101.85:80 [proto: 291.265/MpegDash.AmazonAWS][ClearText][Confidence: DPI][cat: Cloud/13][1 pkts/257 bytes -> 0 pkts/0 bytes][Goodput ratio: 74/0][< 1 sec][Hostname/SNI: livesim.dashif.org][User-Agent: VLC/3.0.16 LibVLC/3.0.16][Risk: ** Known Proto on Non Std Port **][Risk Score: 50][PLAIN TEXT (GET /livesim/sts)][Plen Bins: 0,0,0,0,0,100,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]