Updated README.md (#1562)

* make check great again (not so much)
 * make doc/doc-view
 * CI updates

Signed-off-by: lns <matzeton@googlemail.com>

11 files changed, 93 insertions, 42 deletions

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index b94cdb066..bd16bb1eb 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -30,9 +30,7 @@ jobs:
         run: make all
       - name: Test
         run: |
-          ./tests/do.sh
-          ./tests/do-unit.sh
-          ./tests/do-dga.sh
+          make check VERBOSE=1
       - name: Generate Coverage Report
         run: |
           mkdir -vp coverage_report/ndpi_coverage_report
@@ -66,9 +64,9 @@ jobs:
           sudo make install
       - name: Generate Python bindings
         run: |
-          cd python
           pip install --upgrade pip
-          pip install -r requirements.txt
+          pip install -r python/requirements.txt
+          cd python
           python setup.py install
           cd ..
       - name: Test Python Bindings
@@ -96,11 +94,9 @@ jobs:
           ./autogen.sh --with-only-libndpi
       - name: Generate Documentation
         run: |
-          cd doc
           pip install --upgrade pip
-          pip install -r requirements.txt
-          make html
-          cd ..
+          pip install -r doc/requirements.txt
+          make doc
           mkdir -vp doc/_build/ndpi-documentation-upload/ndpi-documentation
           mv -v doc/_build/html doc/_build/ndpi-documentation-upload/ndpi-documentation/html
       - uses: actions/upload-artifact@v2
@@ -288,9 +284,7 @@ jobs:
       - name: Installing MacOS prerequisites
         if: startsWith(matrix.os, 'macOS') && startsWith(matrix.arch, 'x86_64')
         run: |
-          brew install autoconf automake libtool pkg-config gettext json-c
-          brew install coreutils wdiff colordiff
-          brew install rrdtool
+          brew install coreutils wdiff colordiff autoconf automake libtool pkg-config gettext json-c rrdtool
       - name: Install MacOS Prerequisites (libgcrypt)
         if: startsWith(matrix.os, 'macOS') && startsWith(matrix.arch, 'x86_64') && startsWith(matrix.gcrypt, '--with-local-libgcrypt')
         run: |
@@ -396,8 +390,7 @@ jobs:
             make all &&
             make -C example ndpiSimpleIntegration &&
             make -C rrdtool &&
-            ./tests/do.sh &&
-            ./tests/do-unit.sh
+            make check VERBOSE=1
             "
       - name: Display qemu specified architecture (armhf - little endian)
         if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.arch, 'armhf')
@@ -421,8 +414,7 @@ jobs:
             make all &&
             make -C example ndpiSimpleIntegration &&
             make -C rrdtool &&
-            ./tests/do.sh &&
-            ./tests/do-unit.sh
+            make check VERBOSE=1
             "
       - name: Display qemu specified architecture (s390x - big endian)
         if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.arch, 's390x')
@@ -446,6 +438,5 @@ jobs:
             make all &&
             make -C example ndpiSimpleIntegration &&
             make -C rrdtool &&
-            ./tests/do.sh &&
-            ./tests/do-unit.sh
+            make check VERBOSE=1
             "
diff --git a/Makefile.am b/Makefile.am
index fe9ccc27c..463c043c3 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -21,5 +21,12 @@ EXTRA_DIST = README.md README.nDPI README.fuzzer.md CHANGELOG.md CONTRIBUTING.md
 	sonar-project.properties .github .ci-ignore .lgtm/cpp-queries/packet-payload-integer-arithmetic.ql lgtm.yml
 
 
+.PHONY: doc doc-view
+doc:
+	make -C doc html
+
+doc-view:
+	python3 -m http.server --bind 127.0.0.1 --directory doc/_build/html 8080
+
 changelog:
 	git log --since={`curl -s https://github.com/ntop/ndpi/releases | grep datetime | head -n1 | egrep -o "[0-9]+\-[0-9]+\-[0-9]+"`} --name-only --pretty=format:" - %s" | grep  "^ " > CHANGELOG.latest
diff --git a/README.md b/README.md
index 8c76d1de3..1415908d0 100644
--- a/README.md
+++ b/README.md
@@ -12,44 +12,70 @@ nDPI® is an open source LGPLv3 library for deep-packet inspection. Based on Ope
 
 ### How To Compile nDPI
 
-In order to compile this library do
+In order to compile this project do
 
 - ./autogen.sh
-- ./configure
+- make
+
+To compile the library w/o any tools or tests:
+
+- ./autogen.sh --with-only-libndpi
 - make
 
 To run tests do additionally:
 
-- cd tests; ./do.sh
+- ./tests/do.sh # Generate and check for diff's in PCAP files
+- ./tests/do-unit.sh # Run unit tests
+- ./tests/do-dga.sh # Run DGA detection test
+
+or run all with: `make check`
 
-Please note that the pre-requisites for compilation include:
-- GNU tools (autogen, automake, autoconf, libtool)
-- GNU C compiler (gcc)
+Please note that the (minimal) pre-requisites for compilation include:
+- GNU tools (autoconf automake libtool pkg-config gettext flex bison)
+- GNU C compiler (gcc) or Clang
 
 On Debian/Ubuntu systems do:
-- sudo apt-get install build-essential git bison flex libpcap-dev libtool libtool-bin autoconf pkg-config automake autogen libjson-c-dev libnuma-dev libgcrypt20-dev libpcre2-dev
+- sudo apt-get install build-essential git gettext flex bison libtool autoconf automake pkg-config libpcap-dev libjson-c-dev libnuma-dev libpcre2-dev libmaxminddb-dev librrd-dev
+
+On Arch Linux:
+- sudo pacman -S gcc git gettext flex bison libtool autoconf automake pkg-config libpcap json-c numactl pcre2 libmaxminddb rrdtool
+
+On FreeBSD:
+- sudo pkg install gcc git gettext flex bison libtool autoconf automake devel/pkgconf gmake libpcap json-c pcre2 libmaxminddb rrdtool
+
+Remember to use `gmake` and not `make` on FreeBSD
 
-FreeBSD
-- sudo pkg install gcc git autoconf automake libtool devel/pkgconf json-c gmake
+On MacOS:
+- brew install coreutils gcc git gettext flex bison libtool autoconf automake pkg-config libpcap json-c pcre2 libmaxminddb rrdtool
 
-Remember to use gmake and not make on FreeBSD
+On Windows (assuming [MSYS2](https://www.msys2.org/) already installed):
+- msys2 -c "pacman --noconfirm -S --needed --overwrite '\*' git mingw-w64-x86\_64-toolchain automake1.16 automake-wrapper autoconf libtool make mingw-w64-x86\_64-json-c mingw-w64-x86\_64-crt-git mingw-w64-x86\_64-pcre mingw-w64-x86\_64-libpcap"
+
+### How To Build The Documentation
+
+- pip install --upgrade pip
+- pip install -r doc/requirements.txt
+- make doc
+
+Use the builtin python3 webserver to view documentation:
+- make doc-view
 
 ### How To Add A New Protocol Dissector
 
 The entire procedure of adding new protocols in detail:
 
-1. Add new protocol together with its unique ID to: src/include/ndpi_protocol_ids.h
-2. Create a new protocol in: src/lib/protocols/
-3. Variables to be kept for the duration of the entire flow (as state variables) need to be placed in: src/include/ndpi_typedefs.h in ndpi_flow_tcp_struct (for TCP only), ndpi_flow_udp_struct (for UDP only), or ndpi_flow_struct (for both).
-4. Add a new entry for the search function for the new protocol in: src/include/ndpi_protocols.h
-5. Choose (do not change anything) a selection bitmask from: src/include/ndpi_define.h
-6. Add a new entry in ndpi_set_protocol_detection_bitmask2 in: src/lib/ndpi_main.c
-7. Set protocol default ports in ndpi_init_protocol_defaults in: src/lib/ndpi_main.c
-8.  ./autogen.sh
-9. make
-10. make check
+1. Add new protocol together with its unique ID to: `src/include/ndpi_protocol_ids.h`
+2. Create a new protocol in: `src/lib/protocols/`
+3. Variables to be kept for the duration of the entire flow (as state variables) need to be placed in: `src/include/ndpi_typedefs.h` in `ndpi_flow_tcp_struct` (for TCP only), `ndpi_flow_udp_struct` (for UDP only), or `ndpi_flow_struct` (for both).
+4. Add a new entry for the search function for the new protocol in: `src/include/ndpi_protocols.h`
+5. Choose (do not change anything) a selection bitmask from: `src/include/ndpi_define.h`
+6. Set protocol default ports in `ndpi_init_protocol_defaults` in: `src/lib/ndpi_main.c`
+7. `./autogen.sh`
+8. `make`
+9. `make check`
 
 ### How to use nDPI to Block Selected Traffic
+
 You can use nDPI to selectively block selected Internet traffic by embedding it onto an application (remember that nDPI is just a library). Both [ntopng](https://github.com/ntop/ntopng) and [nProbe cento](http://www.ntop.org/products/netflow/nprobe-cento/) can do this.
 
 ### nDPI Paper Citation
@@ -57,9 +83,12 @@ You can use nDPI to selectively block selected Internet traffic by embedding it
 - Deri, Luca, et al. [nDPI: Open-source high-speed deep packet inspection](http://luca.ntop.org/nDPI.pdf) 2014 International Wireless Communications and Mobile Computing Conference (IWCMC). IEEE, 2014.
 
 ### nDPI-Related Projects
+
 - [nfstream](https://github.com/aouinizied/nfstream)
+- [nDPId](https://github.com/utoni/nDPId)
 
 ### DISCLAIMER
+
 While we do our best to detect network protocols, we cannot guarantee that our software is error free and 100% accurate in protocol detection. Please make sure that you respect the privacy of users and you have proper authorization to listen, capture and inspect network traffic.
 
 nDPI is a registered trademark in the US and EU.
diff --git a/configure.ac b/configure.ac
index 895f1260e..2afd98181 100644
--- a/configure.ac
+++ b/configure.ac
@@ -102,7 +102,10 @@ if ! test "${with_only_libndpi+set}" = set; then :
         JSONC_LIBS="${pkg_cv_JSONC_LIBS}"
         JSONC_CFLAGS="${pkg_cv_JSONC_CFLAGS}"
     ], [AC_MSG_WARN([JSON-C not available. Disabled unit test.])])
-    AC_CHECK_LIB([json-c], [json_object_put], [EXTRA_TARGETS="$EXTRA_TARGETS tests/unit"], [
+    AC_CHECK_LIB([json-c], [json_object_put], [
+        EXTRA_TARGETS="$EXTRA_TARGETS tests/unit"
+        build_unittests=yes
+    ],[
         AC_MSG_WARN([JSON-C not available. Disabled unit test.])
         JSONC_LIBS=""
         JSONC_CFLAGS=""
@@ -114,6 +117,7 @@ if ! test "${with_only_libndpi+set}" = set; then :
         EXTRA_TARGETS=""
     ])
 fi
+AM_CONDITIONAL([BUILD_UNITTESTS], [test "x$build_unittests" = "xyes"])
 
 HANDLE_TLS_SIGS="//"
 AS_IF([test "x${enable_tls_sigs}" = "xyes"],[
diff --git a/example/Makefile.in b/example/Makefile.in
index 370eb2b2f..4e14ca9be 100644
--- a/example/Makefile.in
+++ b/example/Makefile.in
@@ -74,6 +74,9 @@ dpdk:
 	$(MAKE) -f Makefile.dpdk
 
 check:
+	true # nothing to do here
+
+cppcheck:
 	 cppcheck --template='{file}:{line}:{severity}:{message}' --quiet --enable=all --force -I$(SRCHOME)/include *.c
 
 clean:
diff --git a/influxdb/Makefile.in b/influxdb/Makefile.in
index 45d326e32..e5378b25a 100644
--- a/influxdb/Makefile.in
+++ b/influxdb/Makefile.in
@@ -23,3 +23,6 @@ distdir:
 
 install:
 	@echo -n ""
+
+check:
+	true # nothing to do here
diff --git a/rrdtool/Makefile.in b/rrdtool/Makefile.in
index fe585cfa0..228d28eff 100644
--- a/rrdtool/Makefile.in
+++ b/rrdtool/Makefile.in
@@ -26,3 +26,6 @@ distdir:
 
 install:
 	@echo -n ""
+
+check:
+	true # nothing to do here
diff --git a/src/lib/Makefile.in b/src/lib/Makefile.in
index cd8ec0db4..c7cc22283 100644
--- a/src/lib/Makefile.in
+++ b/src/lib/Makefile.in
@@ -85,6 +85,9 @@ distdir:
 distclean: clean
 	/bin/rm -f Makefile
 
+check:
+	true # nothing to do here
+
 cppcheck:
 	cppcheck --template='{file}:{line}:{severity}:{message}' --quiet --enable=all --force -I ../include *.c protocols/*.c
 
diff --git a/tests/Makefile.am b/tests/Makefile.am
index d1fe3730a..64f621ade 100644
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -1,4 +1,8 @@
-TESTS = do.sh
+TESTS = do.sh do-dga.sh
+
+if BUILD_UNITTESTS
+TESTS += do-unit.sh
+endif
 
 EXTRA_DIST = pcap result performance \
 	unit/unit.c unit/Makefile.in \
diff --git a/tests/dga/Makefile.in b/tests/dga/Makefile.in
index efa8b188b..4bd6711bc 100644
--- a/tests/dga/Makefile.in
+++ b/tests/dga/Makefile.in
@@ -36,6 +36,8 @@ install:
 
 distdir:
 
-
 distclean: clean
 	/bin/rm -f Makefile
+
+check:
+	true # nothing to do here, done by invoking tests/do-dga.sh
diff --git a/tests/unit/Makefile.in b/tests/unit/Makefile.in
index e1d13328e..36f8bec3e 100644
--- a/tests/unit/Makefile.in
+++ b/tests/unit/Makefile.in
@@ -60,6 +60,8 @@ install:
 
 distdir:
 
-
 distclean: clean
 	/bin/rm -f Makefile
+
+check:
+	true # nothing to do here, done by invoking tests/do-unit.sh