Improve packaging checks (#4188)

* Simplify the packaging ignore list

Applying sed commands to the result of a diff makes it non-obvious
what is being ignored and what not. Shell globbing is much better
understood.

At the same time, the script is now verbose about what happened when
a mismatch occurs, to help fix the detected issue.

* Run release tar comparison on all builds

The checks were only running when a release was imminent or in progress,
now they run every time, so a travis build will now stop if files are
not accounted for.

packaging/git-build

@@ -50,6 +50,5 @@ set -e
 autoreconf -ivf
 ./configure --enable-maintainer-mode 
 set +e
-make dist
-status=$?
-exit $status
+make dist || exit
+./packaging/tar-compare . netdata-*.tar.gz || exit

packaging/packaging.functions

@@ -87,8 +87,6 @@ try_build() {
   git diff --staged | patch -p1 -d "$MYTMP/build"
   (cd $MYTMP/build; ./packaging/git-build || touch $MYTMP/fail)
   if [ -f $MYTMP/fail ]; then return 1; fi
-  (cd $MYTMP/build; ./packaging/tar-compare . *.tar.gz || touch $MYTMP/fail)
-  if [ -f $MYTMP/fail ]; then return 1; fi
   touch $MYTMP/success
   return 0
 }

packaging/tar-compare

@@ -6,6 +6,29 @@
 # lists files which appear in one or the other only, to help check
 # for missing EXTRA_DIST entries in Makefile.am files.
 
+# Note: this list uses shell globbing, i.e. an * matches any number of
+# subdirectories, so it is somewhat different to e.g. .gitignore
+ignore=$(tr '\n' '|' <<'END'
+autom4te.cache/*
+CMakeLists.txt
+config.h
+config.log
+config.status
+.git/*
+.githooks/*
+Makefile
+*/Makefile
+makeself/README.md
+netdata-*.tar.*
+packaging/*
+src/.deps/*
+stamp-h1
+tests/profile/*
+.travis.yml
+web/images/README.md
+END
+)
+
 scriptname=tar-compare
 if ! MYTMP=$(mktemp -d -t $scriptname-XXXXXX)
 then
@@ -29,38 +52,27 @@ trap cleanup 0
 
 if [ $# -ne 2 ]
 then
-  echo "tar-compare git-dir tar-gz-file"
+  echo "Usage: tar-compare git-dir tar-gz-file"
   exit 1
 fi
 
 mkdir $MYTMP/unpack
-tar xfzC "$2" $MYTMP/unpack
-diff -r "$1" $MYTMP/unpack/* | grep "^Only" | sed \
-	-e '/: autom4te\.cache$/d' \
-	-e '/: \.deps$/d' \
-	-e '/: \.git$/d' \
-	-e '/: \.gitattributes$/d' \
-	-e '/: \.gitignore$/d' \
-	-e '/: config\.log$/d' \
-	-e '/: config\.status$/d' \
-	-e '/: config\.h.*$/d' \
-	-e '/: Makefile$/d' \
-	-e '/: .githooks$/d' \
-	-e '/: packaging$/d' \
-	-e '/: stamp-h1$/d' \
-	-e '/: README\.md$/d' \
-	-e '/: tmp-anchor-links$/d' \
-	-e '/: tmp-manproc$/d' \
-	-e '/: .*\.tar\.gz$/d' \
-	-e '/: .*\.tar\.bz2$/d' \
-	-e '/: .*\.tar\.xz$/d' \
-	-e '/: unittest$/d' \
-	-e '/: iprange$/d' \
-	-e '/: .*\.o$/d' \
-	-e '/: CMakeLists.txt/d' \
-	-e '/: tests$/d' \
-	-e '/: .travis.yml/d' > $MYTMP/out
+tar xfzC "$2" $MYTMP/unpack || exit
+(cd "$1" && find . -type f | sort > $MYTMP/git-list)
+(cd "$MYTMP/unpack"/* && find . -type f | sort > $MYTMP/tar-list)
+
+comm -23 $MYTMP/git-list $MYTMP/tar-list | cut -f2- -d'/' > $MYTMP/only-in-git
+
+while read name
+do
+  eval "case $name in $ignore,) : ;; *) echo 'In git but not tar: $name' >> '$MYTMP/out' ;; esac"
+done < $MYTMP/only-in-git
 
-cat $MYTMP/out
-test -s $MYTMP/out && exit 1
+if [ -s $MYTMP/out ]
+then
+  cat $MYTMP/out
+  echo "Release tarfile differs from git content unexpectedly."
+  echo "Ensure all files are packaged, or ignored by packaging/tar-compare"
+  exit 1
+fi
 exit 0