Added script to compare and verify the output of `make dist'. (#1551)

This fixes some build/test issues resulting when using tarballs.

 * nDPI uses autotools (especially autoconf) in a wrong way, see #1163

Signed-off-by: lns <matzeton@googlemail.com>

8 files changed, 62 insertions, 8 deletions

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 91e45280f..b4f293859 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -329,10 +329,11 @@ jobs:
         if: startsWith(matrix.arch, 'x86_64') && startsWith(matrix.os, 'windows')
         run: |
           msys2 -c './tests/do-dga.sh'
-      - name: Generate tarball
+      - name: Generate/Verify tarball
         if: startsWith(matrix.os, 'ubuntu-latest') && startsWith(matrix.arch, 'x86_64')
         run: |
           make dist
+          ./utils/verify_dist_tarball.sh
       - name: Generate documentation
         if: startsWith(matrix.os, 'ubuntu-latest') && startsWith(matrix.arch, 'x86_64')
         run: |
diff --git a/Makefile.am b/Makefile.am
index 86f53fff7..1b6ea524e 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -8,7 +8,18 @@ endif
 pkgconfigdir = $(libdir)/pkgconfig
 pkgconfig_DATA = libndpi.pc
 
-EXTRA_DIST = README.md CHANGELOG.md CONTRIBUTING.md autogen.sh wireshark windows utils packages doc/api.rst doc/conf.py doc/flow_risks.rst doc/guide/nDPI_QuickStartGuide.pages doc/guide/nDPI_QuickStartGuide.pdf doc/img/logo.png doc/index.rst doc/Makefile doc/what_is_ndpi.rst
+EXTRA_DIST = README.md README.nDPI README.fuzzer.md CHANGELOG.md CONTRIBUTING.md \
+	autogen.sh wireshark windows utils packages \
+	influxdb/Makefile.in influxdb/metric_anomaly.c influxdb/README.txt \
+	rrdtool/Makefile.in rrdtool/README.txt rrdtool/rrd_anomaly.c rrdtool/rrd_similarity.c \
+	doc/api.rst doc/conf.py doc/flow_risks.rst doc/guide/nDPI_QuickStartGuide.pages \
+		doc/guide/nDPI_QuickStartGuide.pdf doc/img/logo.png doc/index.rst \
+		doc/Makefile doc/what_is_ndpi.rst \
+	python/DEV_GUIDE.md python/dev_requirements.txt python/ndpi_example.py python/ndpi/__init__.py \
+		python/ndpi/ndpi_build.py python/ndpi/ndpi.py python/README.md \
+		python/requirements.txt python/setup.py python/tests.py \
+	sonar-project.properties .github .lgtm/cpp-queries/packet-payload-integer-arithmetic.ql lgtm.yml
+
 
 changelog:
 	git log --since={`curl -s https://github.com/ntop/ndpi/releases | grep datetime | head -n1 | egrep -o "[0-9]+\-[0-9]+\-[0-9]+"`} --name-only --pretty=format:" - %s" | grep  "^ " > CHANGELOG.latest
diff --git a/example/Makefile.in b/example/Makefile.in
index a8b2b0a5b..370eb2b2f 100644
--- a/example/Makefile.in
+++ b/example/Makefile.in
@@ -83,8 +83,12 @@ clean:
 	/bin/rm -rf build
 
 distdir:
-	cp categories.txt mining_hosts.txt protos.txt README.DPDK '$(distdir)/'
-	find . -maxdepth 1 -type f -name '*.c' -o -name '*.h' -o -name '*.py' | xargs -I'{}' cp '{}' '$(distdir)/{}'
+	cp README.DPDK '$(distdir)/'
+	find . -maxdepth 1 -type f -name '*.c' \
+		-o -name '*.h' \
+		-o -name '*.py' \
+		-o -name '*.txt' \
+		-o -name '*.csv' | xargs -I'{}' cp '{}' '$(distdir)/{}'
 
 distclean: clean
 	/bin/rm -f Makefile.dpdk
diff --git a/influxdb/Makefile.in b/influxdb/Makefile.in
index 17ea45c53..45d326e32 100644
--- a/influxdb/Makefile.in
+++ b/influxdb/Makefile.in
@@ -18,6 +18,8 @@ distclean: clean
 	/bin/rm -f Makefile
 
 distdir:
+	find . -type f -name '*.c' \
+		-o -name '*.txt' | xargs -I'{}' cp '{}' '$(distdir)/{}'
 
 install:
 	@echo -n ""
diff --git a/rrdtool/Makefile.in b/rrdtool/Makefile.in
index 2b7e615c1..fe585cfa0 100644
--- a/rrdtool/Makefile.in
+++ b/rrdtool/Makefile.in
@@ -21,6 +21,8 @@ distclean: clean
 	/bin/rm -f Makefile
 
 distdir:
+	find . -type f -name '*.c' \
+		-o -name '*.txt' | xargs -I'{}' cp '{}' '$(distdir)/{}'
 
 install:
 	@echo -n ""
diff --git a/src/lib/Makefile.in b/src/lib/Makefile.in
index 00464286d..cd8ec0db4 100644
--- a/src/lib/Makefile.in
+++ b/src/lib/Makefile.in
@@ -72,10 +72,15 @@ clean:
 	/bin/rm -f $(NDPI_LIB_STATIC) $(OBJECTS) *.o *.so *.lo libndpi.so*
 
 distdir:
-	cp ndpi_content_match.c.inc '$(distdir)/'
 	find . -type d | xargs -I'{}' mkdir -p '$(distdir)/{}'
-	find ../include -type f -name '*.h' | xargs -I'{}' cp '{}' '$(distdir)/{}'
-	find . -type f -name '*.c' -o -name '*.h' | xargs -I'{}' cp '{}' '$(distdir)/{}'
+	find ../include -type f -name '*.h' \
+		-a '!' -name 'ndpi_api.h' \
+		-a '!' -name 'ndpi_config.h' \
+		-a '!' -name 'ndpi_define.h' | xargs -I'{}' cp '{}' '$(distdir)/{}'
+	find . -type f -name '*.c' \
+		-o -name '*.cc' \
+		-o -name '*.c.inc' \
+		-o -name '*.h' | xargs -I'{}' cp '{}' '$(distdir)/{}'
 
 distclean: clean
 	/bin/rm -f Makefile
diff --git a/tests/Makefile.am b/tests/Makefile.am
index 271516b87..d1fe3730a 100644
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -1,6 +1,9 @@
 TESTS = do.sh
 
-EXTRA_DIST = ossfuzz.sh pcap result
+EXTRA_DIST = pcap result performance \
+	unit/unit.c unit/Makefile.in \
+	dga/dga_evaluate.c dga/Makefile.in dga/README.md dga/test_dga.csv dga/test_non_dga.csv do-dga.sh \
+	do-unit.sh do.sh.in ossfuzz.sh
 
 all:
 	@echo -n ""
diff --git a/utils/verify_dist_tarball.sh b/utils/verify_dist_tarball.sh
new file mode 100755
index 000000000..0bfbe71c1
--- /dev/null
+++ b/utils/verify_dist_tarball.sh
@@ -0,0 +1,26 @@
+#!/usr/bin/env sh
+
+EXCLUDE_PATTERN="(.*\.m4$|Makefile$|Makefile\.in$|utils/verify_dist_tarball\.sh|^packages/debian/.*|^packages/debian|^test-driver|^config\.guess|^config\.sub|^compile|^configure|/|depcomp|.gitattributes|.gitignore|install-sh|ltmain.sh|missing|src/include/ndpi_config\.h\.in)$"
+
+set -x
+set -e
+
+cd "$(dirname "${0}")/.."
+
+git ls-tree --full-tree --name-only -r HEAD | grep -vE "${EXCLUDE_PATTERN}" | sort >/tmp/ndpi-dist-verify-git.txt
+
+TARBALL="${1}"
+if [ "x${TARBALL}" = x ]; then
+	if [ ! -r Makefile ]; then
+		./autogen.sh
+	fi
+	make dist
+	AC_VERSION="$(sed -n 's/^AC_INIT.*\([[:digit:]]\+\.[[:digit:]]\+\.[[:digit:]]\+\).*$/\1/gp' < configure.ac)"
+	TARBALL="./libndpi-${AC_VERSION}.tar.gz"
+fi
+
+tar -tzf "${TARBALL}" | sed -n 's|^[^/]*/||gp' | grep -v '^$' | grep -vE "${EXCLUDE_PATTERN}" | sort >/tmp/ndpi-dist-verify-tar.txt
+
+diff -u0 /tmp/ndpi-dist-verify-git.txt /tmp/ndpi-dist-verify-tar.txt
+
+rm -f /tmp/ndpi-dist-verify-git.txt /tmp/ndpi-dist-verify-tar.txt