Added script to compare and verify the output of `make dist'. (#1551)

This fixes some build/test issues resulting when using tarballs.

 * nDPI uses autotools (especially autoconf) in a wrong way, see #1163

Signed-off-by: lns <matzeton@googlemail.com>

.github/workflows/build.yml

@@ -329,10 +329,11 @@ jobs:
         if: startsWith(matrix.arch, 'x86_64') && startsWith(matrix.os, 'windows')
         run: |
           msys2 -c './tests/do-dga.sh'
-      - name: Generate tarball
+      - name: Generate/Verify tarball
         if: startsWith(matrix.os, 'ubuntu-latest') && startsWith(matrix.arch, 'x86_64')
         run: |
           make dist
+          ./utils/verify_dist_tarball.sh
       - name: Generate documentation
         if: startsWith(matrix.os, 'ubuntu-latest') && startsWith(matrix.arch, 'x86_64')
         run: |

Makefile.am

@@ -8,7 +8,18 @@ endif
 pkgconfigdir = $(libdir)/pkgconfig
 pkgconfig_DATA = libndpi.pc
 
-EXTRA_DIST = README.md CHANGELOG.md CONTRIBUTING.md autogen.sh wireshark windows utils packages doc/api.rst doc/conf.py doc/flow_risks.rst doc/guide/nDPI_QuickStartGuide.pages doc/guide/nDPI_QuickStartGuide.pdf doc/img/logo.png doc/index.rst doc/Makefile doc/what_is_ndpi.rst
+EXTRA_DIST = README.md README.nDPI README.fuzzer.md CHANGELOG.md CONTRIBUTING.md \
+	autogen.sh wireshark windows utils packages \
+	influxdb/Makefile.in influxdb/metric_anomaly.c influxdb/README.txt \
+	rrdtool/Makefile.in rrdtool/README.txt rrdtool/rrd_anomaly.c rrdtool/rrd_similarity.c \
+	doc/api.rst doc/conf.py doc/flow_risks.rst doc/guide/nDPI_QuickStartGuide.pages \
+		doc/guide/nDPI_QuickStartGuide.pdf doc/img/logo.png doc/index.rst \
+		doc/Makefile doc/what_is_ndpi.rst \
+	python/DEV_GUIDE.md python/dev_requirements.txt python/ndpi_example.py python/ndpi/__init__.py \
+		python/ndpi/ndpi_build.py python/ndpi/ndpi.py python/README.md \
+		python/requirements.txt python/setup.py python/tests.py \
+	sonar-project.properties .github .lgtm/cpp-queries/packet-payload-integer-arithmetic.ql lgtm.yml
+
 
 changelog:
 	git log --since={`curl -s https://github.com/ntop/ndpi/releases | grep datetime | head -n1 | egrep -o "[0-9]+\-[0-9]+\-[0-9]+"`} --name-only --pretty=format:" - %s" | grep  "^ " > CHANGELOG.latest

example/Makefile.in

@@ -83,8 +83,12 @@ clean:
 	/bin/rm -rf build
 
 distdir:
-	cp categories.txt mining_hosts.txt protos.txt README.DPDK '$(distdir)/'
-	find . -maxdepth 1 -type f -name '*.c' -o -name '*.h' -o -name '*.py' | xargs -I'{}' cp '{}' '$(distdir)/{}'
+	cp README.DPDK '$(distdir)/'
+	find . -maxdepth 1 -type f -name '*.c' \
+		-o -name '*.h' \
+		-o -name '*.py' \
+		-o -name '*.txt' \
+		-o -name '*.csv' | xargs -I'{}' cp '{}' '$(distdir)/{}'
 
 distclean: clean
 	/bin/rm -f Makefile.dpdk

influxdb/Makefile.in

@@ -18,6 +18,8 @@ distclean: clean
 	/bin/rm -f Makefile
 
 distdir:
+	find . -type f -name '*.c' \
+		-o -name '*.txt' | xargs -I'{}' cp '{}' '$(distdir)/{}'
 
 install:
 	@echo -n ""

rrdtool/Makefile.in

@@ -21,6 +21,8 @@ distclean: clean
 	/bin/rm -f Makefile
 
 distdir:
+	find . -type f -name '*.c' \
+		-o -name '*.txt' | xargs -I'{}' cp '{}' '$(distdir)/{}'
 
 install:
 	@echo -n ""

src/lib/Makefile.in

@@ -72,10 +72,15 @@ clean:
 	/bin/rm -f $(NDPI_LIB_STATIC) $(OBJECTS) *.o *.so *.lo libndpi.so*
 
 distdir:
-	cp ndpi_content_match.c.inc '$(distdir)/'
 	find . -type d | xargs -I'{}' mkdir -p '$(distdir)/{}'
-	find ../include -type f -name '*.h' | xargs -I'{}' cp '{}' '$(distdir)/{}'
-	find . -type f -name '*.c' -o -name '*.h' | xargs -I'{}' cp '{}' '$(distdir)/{}'
+	find ../include -type f -name '*.h' \
+		-a '!' -name 'ndpi_api.h' \
+		-a '!' -name 'ndpi_config.h' \
+		-a '!' -name 'ndpi_define.h' | xargs -I'{}' cp '{}' '$(distdir)/{}'
+	find . -type f -name '*.c' \
+		-o -name '*.cc' \
+		-o -name '*.c.inc' \
+		-o -name '*.h' | xargs -I'{}' cp '{}' '$(distdir)/{}'
 
 distclean: clean
 	/bin/rm -f Makefile

tests/Makefile.am

@@ -1,6 +1,9 @@
 TESTS = do.sh
 
-EXTRA_DIST = ossfuzz.sh pcap result
+EXTRA_DIST = pcap result performance \
+	unit/unit.c unit/Makefile.in \
+	dga/dga_evaluate.c dga/Makefile.in dga/README.md dga/test_dga.csv dga/test_non_dga.csv do-dga.sh \
+	do-unit.sh do.sh.in ossfuzz.sh
 
 all:
 	@echo -n ""

utils/verify_dist_tarball.sh

@@ -0,0 +1,26 @@
+#!/usr/bin/env sh
+
+EXCLUDE_PATTERN="(.*\.m4$|Makefile$|Makefile\.in$|utils/verify_dist_tarball\.sh|^packages/debian/.*|^packages/debian|^test-driver|^config\.guess|^config\.sub|^compile|^configure|/|depcomp|.gitattributes|.gitignore|install-sh|ltmain.sh|missing|src/include/ndpi_config\.h\.in)$"
+
+set -x
+set -e
+
+cd "$(dirname "${0}")/.."
+
+git ls-tree --full-tree --name-only -r HEAD | grep -vE "${EXCLUDE_PATTERN}" | sort >/tmp/ndpi-dist-verify-git.txt
+
+TARBALL="${1}"
+if [ "x${TARBALL}" = x ]; then
+	if [ ! -r Makefile ]; then
+		./autogen.sh
+	fi
+	make dist
+	AC_VERSION="$(sed -n 's/^AC_INIT.*\([[:digit:]]\+\.[[:digit:]]\+\.[[:digit:]]\+\).*$/\1/gp' < configure.ac)"
+	TARBALL="./libndpi-${AC_VERSION}.tar.gz"
+fi
+
+tar -tzf "${TARBALL}" | sed -n 's|^[^/]*/||gp' | grep -v '^$' | grep -vE "${EXCLUDE_PATTERN}" | sort >/tmp/ndpi-dist-verify-tar.txt
+
+diff -u0 /tmp/ndpi-dist-verify-git.txt /tmp/ndpi-dist-verify-tar.txt
+
+rm -f /tmp/ndpi-dist-verify-git.txt /tmp/ndpi-dist-verify-tar.txt