From a9cc98b4c4c9454c0444904686992e0d67876b5c Mon Sep 17 00:00:00 2001
From: Jonathan Perkin <jperkin@joyent.com>
Date: Fri, 16 Oct 2015 14:01:21 +0100
Subject: [PATCH 01/28] Ignore files under PKGBUILD_BASEDIR for scanning.

These update a lot but not in a way that will affect the scan results,
and if that is the case then we remove old bulklogs anyway.
---
 mk/pbulk/pbulk-index.mk | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mk/pbulk/pbulk-index.mk b/mk/pbulk/pbulk-index.mk
index 0c76d113486a..6c634bb352e9 100644
--- a/mk/pbulk/pbulk-index.mk
+++ b/mk/pbulk/pbulk-index.mk
@@ -117,7 +117,7 @@ pbulk-index-item:
 	@echo "USE_DESTDIR="${_USE_DESTDIR:Q}
 	@echo "BOOTSTRAP_PKG="${BOOTSTRAP_PKG}
 	@echo "USERGROUP_PHASE="${USERGROUP_PHASE:Q}
-	@echo "SCAN_DEPENDS="${_PBULK_SCAN_DEPENDS:O:u:Q}
+	@echo "SCAN_DEPENDS="${_PBULK_SCAN_DEPENDS:N${PKGBUILD_BASEDIR:U/nonexistent}/*:O:u:Q}
 .if defined(_PBULK_MULTI_NEEDED)
 	@printf "MULTI_VERSION="
 .for _t in ${_PBULK_MULTI_NEEDED}

From 623eff4b8b6c1056fe31ea416f7c9950b00207ac Mon Sep 17 00:00:00 2001
From: Jonathan Perkin <jperkin@joyent.com>
Date: Fri, 16 Oct 2015 14:18:53 +0100
Subject: [PATCH 02/28] Support chrooted builds.

This adds a chroot handling script and a few additional variables to
pbulk.conf:

  - $chroot_create and $chroot_delete point to scripts which create and
    delete a sandbox respectively, each taking a path argument of the
    chroot location.

  - $chroot_dir is the base directory where chroots are created.

  - $build_chroots and $scan_chroots specify the number of build and
    scan chroots to create respectively.

If the number of chroots is 1, they are named:

  ${chroot_dir}-build
  ${chroot_dir}-scan

otherwise they are named i=1..N:

  ${chroot_dir}-build-${i}
  ${chroot_dir}-scan-${i}

The chroot_dir variable isn't brilliantly named, but at this point is
baked into too many of our pbulk branches to change.

Currently users of this need to provide their own create/delete scripts,
though hopefully at some point I'll provide sysutils/mksandbox wrappers
as examples for getting started.
---
 pkgtools/pbulk/Makefile                       |  2 +-
 pkgtools/pbulk/PLIST                          |  1 +
 pkgtools/pbulk/files/pbulk/pbulk.conf         | 18 ++++
 pkgtools/pbulk/files/pbulk/scripts/Makefile   |  2 +-
 pkgtools/pbulk/files/pbulk/scripts/build      |  2 +
 .../files/pbulk/scripts/build-client-start    |  8 +-
 pkgtools/pbulk/files/pbulk/scripts/chroot     | 90 +++++++++++++++++++
 .../pbulk/files/pbulk/scripts/client-prepare  |  8 +-
 pkgtools/pbulk/files/pbulk/scripts/scan       |  4 +
 .../files/pbulk/scripts/scan-client-start     |  8 +-
 10 files changed, 138 insertions(+), 5 deletions(-)
 create mode 100755 pkgtools/pbulk/files/pbulk/scripts/chroot

diff --git a/pkgtools/pbulk/Makefile b/pkgtools/pbulk/Makefile
index 62582a3caca0..68e4d6b52c45 100644
--- a/pkgtools/pbulk/Makefile
+++ b/pkgtools/pbulk/Makefile
@@ -25,7 +25,7 @@ SUBST_STAGE.tools=	pre-configure
 SUBST_MESSAGE.tools=	Fixing references to tools
 SUBST_FILES.tools=	pbulk.conf scripts/build scripts/build-client-start \
 	scripts/bulkbuild scripts/bulkbuild-rebuild scripts/bulkbuild-restart \
-	scripts/client-clean scripts/client-prepare \
+	scripts/chroot scripts/client-clean scripts/client-prepare \
 	scripts/pkg-build scripts/pkg-up-to-date scripts/pre-build \
 	scripts/report scripts/scan scripts/scan-client-start scripts/upload \
 	scripts/compute-packages.awk scripts/create-broken-graph.awk \
diff --git a/pkgtools/pbulk/PLIST b/pkgtools/pbulk/PLIST
index 664954455847..c4097b0f1a6e 100644
--- a/pkgtools/pbulk/PLIST
+++ b/pkgtools/pbulk/PLIST
@@ -4,6 +4,7 @@ bin/bulkbuild-rebuild
 bin/bulkbuild-restart
 libexec/pbulk/build
 libexec/pbulk/build-client-start
+libexec/pbulk/chroot
 libexec/pbulk/client-clean
 libexec/pbulk/client-prepare
 libexec/pbulk/compute-packages
diff --git a/pkgtools/pbulk/files/pbulk/pbulk.conf b/pkgtools/pbulk/files/pbulk/pbulk.conf
index 5a79692dd43d..bff42082a5a3 100644
--- a/pkgtools/pbulk/files/pbulk/pbulk.conf
+++ b/pkgtools/pbulk/files/pbulk/pbulk.conf
@@ -26,6 +26,23 @@ build_clients="192.168.75.21 192.168.75.22 192.168.75.23 192.168.75.24"
 master_port_scan=${master_ip}:2001
 master_port_build=${master_ip}:2002
 
+# Create chroots for each phase.  This allows concurrent build/scan on the
+# same host.  chroot_create/chroot_delete are scripts which take a single
+# argument which is the chroot directory to create/delete, and chroot_dir is
+# used as the prefix for the name of the chroot directory.
+#
+# To use chroots on a single host, set master_mode=yes and leave both
+# build_clients and scan_clients unset or empty.
+#
+#chroot_create=/path/to/mksandbox
+#chroot_delete=/path/to/rmsandbox
+#chroot_dir=/chroot/pkgsrc
+
+# To enable concurrent chroots, set build_chroots/scan_chroots > 1.
+#
+#build_chroots=4
+#scan_chroots=8
+
 # Some flags for the programs that publish the binary packages and the
 # build report. If you want to disable the publishing at all, see below.
 #
@@ -119,6 +136,7 @@ keep_prefix=no
 client_prepare=@PREFIX@/libexec/pbulk/client-prepare
 client_prepare_action=@PREFIX@/libexec/pbulk/client-clean
 client_prepare_chroot=:
+chroot_script=@PREFIX@/libexec/pbulk/chroot
 pkg_up_to_date_script=@PREFIX@/libexec/pbulk/pkg-up-to-date
 pbuild_script=@PREFIX@/libexec/pbulk/pkg-build
 pbuild_start_script=@PREFIX@/libexec/pbulk/build-client-start
diff --git a/pkgtools/pbulk/files/pbulk/scripts/Makefile b/pkgtools/pbulk/files/pbulk/scripts/Makefile
index 057e5b9455ec..d904a6b9fb3b 100644
--- a/pkgtools/pbulk/files/pbulk/scripts/Makefile
+++ b/pkgtools/pbulk/files/pbulk/scripts/Makefile
@@ -2,7 +2,7 @@
 
 SCRIPTS=	build build-client-start bulkbuild bulkbuild-rebuild \
 		bulkbuild-restart \
-		client-clean client-prepare \
+		chroot client-clean client-prepare \
 		pkg-build pkg-up-to-date pre-build report \
 		scan scan-client-start upload \
 		compute-packages.awk create-broken-graph.awk \
diff --git a/pkgtools/pbulk/files/pbulk/scripts/build b/pkgtools/pbulk/files/pbulk/scripts/build
index 1a0f9ed97a67..494e8574bec5 100755
--- a/pkgtools/pbulk/files/pbulk/scripts/build
+++ b/pkgtools/pbulk/files/pbulk/scripts/build
@@ -46,7 +46,9 @@ case "${master_mode}" in
 	${pbuild} -r ${loc}/pbuild -v -b ${pbuild_script} ${loc}/presolve ${loc}/success ${loc}/error
 	;;
 [yY][eE][sS])
+	[ -n "${chroot_dir}" ] && ${chroot_script} build create ${chroot_dir}
 	${pbuild} -r ${loc}/pbuild -I ${pbuild_start_script} -m ${master_port_build} -v ${loc}/presolve ${loc}/success ${loc}/error
+	[ -n "${chroot_dir}" ] && ${chroot_script} build delete ${chroot_dir}
 	;;
 *)
 	echo "master_mode must be either yes or no."
diff --git a/pkgtools/pbulk/files/pbulk/scripts/build-client-start b/pkgtools/pbulk/files/pbulk/scripts/build-client-start
index 770569482477..c557044e9507 100755
--- a/pkgtools/pbulk/files/pbulk/scripts/build-client-start
+++ b/pkgtools/pbulk/files/pbulk/scripts/build-client-start
@@ -29,7 +29,13 @@ for client in ${build_clients}; do
 		;;
 	esac
 	if [ -z "$path" ]; then
-		ssh $port $client "${pbuild} -c ${master_port_build} -b ${pbuild_script}" &
+		if [ -n "${chroot_create}" -a "${chroot_dir}" ]; then
+			ssh $port $client "for dir in ${chroot_dir}-build*; do
+				PBULK_CONF=${PBULK_CONF} chroot \${dir} @SH@ -c \"${pbuild} -c ${master_port_build} -b ${pbuild_script}\" &
+			done" &
+		else
+			ssh $port $client "${pbuild} -c ${master_port_build} -b ${pbuild_script}" &
+		fi
 	else
 		chroot "$path" ${pbuild} -c ${master_port_build} -b ${pbuild_script} &
 	fi
diff --git a/pkgtools/pbulk/files/pbulk/scripts/chroot b/pkgtools/pbulk/files/pbulk/scripts/chroot
new file mode 100755
index 000000000000..8e1bbc723983
--- /dev/null
+++ b/pkgtools/pbulk/files/pbulk/scripts/chroot
@@ -0,0 +1,90 @@
+#!@SH@
+# $NetBSD$
+#
+# Script to manage creating/deleting chroots during the pbulk process
+#
+# Usage: chroot <build|scan> <create|delete> <dir>
+#
+
+. ${PBULK_CONF:-@PBULK_CONFIG@}
+
+set -e
+
+if [ "${config_version}" != "@PBULK_CONFIG_VERSION@" ]; then
+	echo "Your configuration has version ${config_version}."
+	echo "This version of pbulk expects version @PBULK_CONFIG_VERSION@."
+	exit 1
+fi
+
+phase=$1; shift
+operation=$1; shift
+chroot_dir=$1; shift
+
+case "${phase}" in
+build)
+	clients=${build_clients}
+	numchroots=${build_chroots}
+	;;
+scan)
+	clients=${scan_clients}
+	numchroots=${scan_chroots}
+	;;
+*)
+	echo "$0: Unexpected phase '${phase}'" >&2
+	exit 1
+	;;
+esac
+
+case "${operation}" in
+create)
+	chroot_script=${chroot_create}
+	;;
+delete)
+	chroot_script=${chroot_delete}
+	;;
+*)
+	echo "$0: Unexpected operation '${operation}'" >&2
+	exit 1
+	;;
+esac
+
+if [ -z "${chroot_script}" ] || [ ! -x "${chroot_script}" ]; then
+	echo "$0: chroot_${operation} unset or not found/executable" >&2
+	exit 1
+fi
+
+# Create chroots on remote clients
+if [ -n "${clients}" ]; then
+	for client in ${clients}; do
+		if [ -n "${numchroots}" -a ${numchroots} -gt 1 ]; then
+			ssh $client "
+				i=1
+				while true; do
+					${chroot_script} ${chroot_dir}-${phase}-\${i} &
+					i=\`expr \$i + 1\`
+					if [ \$i -gt ${numchroots} ]; then
+						break
+					fi
+				done" &
+		else
+			ssh $client "${chroot_script} ${chroot_dir}-${phase}" &
+		fi
+	done
+# Create local chroots
+else
+	if [ -n "${numchroots}" -a ${numchroots} -gt 1 ]; then
+		i=1
+		while true; do
+			${chroot_script} ${chroot_dir}-${phase}-${i} &
+			i=`expr $i + 1`
+			if [ $i -gt ${numchroots} ]; then
+				break
+			fi
+		done
+	else
+		${chroot_script} ${chroot_dir}-${phase}
+	fi
+fi
+
+# Ensure all chroot operations are complete before continuing.
+wait
diff --git a/pkgtools/pbulk/files/pbulk/scripts/client-prepare b/pkgtools/pbulk/files/pbulk/scripts/client-prepare
index eace7bd61732..75bb0550c41a 100755
--- a/pkgtools/pbulk/files/pbulk/scripts/client-prepare
+++ b/pkgtools/pbulk/files/pbulk/scripts/client-prepare
@@ -44,7 +44,13 @@ for client in ${unique_clients}; do
 		;;
 	esac
 	if [ -z "$path" ]; then
-		ssh $port $client "${client_prepare_action}" &
+		if [ -n "${chroot_create}" -a "${chroot_dir}" ]; then
+			ssh $port $client "for dir in ${chroot_dir}-scan*; do
+				PBULK_CONF=${PBULK_CONF} chroot \${dir} @SH@ -c \"${client_prepare_action}\" &
+			done" &
+		else
+			ssh $port $client "${client_prepare_action}" &
+		fi
 	else
 		$client_prepare_chroot "$path" &
 	fi
diff --git a/pkgtools/pbulk/files/pbulk/scripts/scan b/pkgtools/pbulk/files/pbulk/scripts/scan
index 8282a31a56b1..d24f0dfc2ea6 100755
--- a/pkgtools/pbulk/files/pbulk/scripts/scan
+++ b/pkgtools/pbulk/files/pbulk/scripts/scan
@@ -54,7 +54,9 @@ if [ -z "${limited_list}" ]; then
 		${pscan} -v -M ${make} ${extra_pscan_args} ${pkgsrc} ${loc}/pscan 2>> ${loc}/pscan.stderr
 		;;
 	[yY][eE][sS])
+		[ -n "${chroot_dir}" ] && ${chroot_script} scan create ${chroot_dir}
 		${pscan} -v -I ${pscan_start_script} -m ${master_port_scan} -M ${make} ${extra_pscan_args} ${pkgsrc} ${loc}/pscan 2>> ${loc}/pscan.stderr
+		[ -n "${chroot_dir}" ] && ${chroot_script} scan delete ${chroot_dir}
 		;;
 	*)
 		echo "master_mode must be either yes or no."
@@ -89,7 +91,9 @@ else
 			${pscan} -v -l -M ${make} ${pkgsrc} ${loc}/pscan < ${loc}/missing.s
 			;;
 		[yY][eE][sS])
+			[ -n "${chroot_dir}" ] && ${chroot_script} scan create ${chroot_dir}
 			${pscan} -v -l -I ${pscan_start_script} -m ${master_port_scan} -M ${make} ${pkgsrc} ${loc}/pscan < ${loc}/missing.s
+			[ -n "${chroot_dir}" ] && ${chroot_script} scan delete ${chroot_dir}
 			;;
 		*)
 			echo "master_mode must be either yes or no."
diff --git a/pkgtools/pbulk/files/pbulk/scripts/scan-client-start b/pkgtools/pbulk/files/pbulk/scripts/scan-client-start
index 158e71f31f58..bcf58077b986 100755
--- a/pkgtools/pbulk/files/pbulk/scripts/scan-client-start
+++ b/pkgtools/pbulk/files/pbulk/scripts/scan-client-start
@@ -37,7 +37,13 @@ for client in ${scan_clients}; do
 		;;
 	esac
 	if [ -z "$path" ]; then
-		ssh $port $client "${pscan} -c ${master_port_scan} -M ${make} ${extra_pscan_args} ${pkgsrc}" &
+		if [ -n "${chroot_create}" -a "${chroot_dir}" ]; then
+			ssh $port $client "for dir in ${chroot_dir}-scan*; do
+				PBULK_CONF=${PBULK_CONF} chroot \${dir} @SH@ -c \"${pscan} -c ${master_port_scan} -M ${make} ${extra_pscan_args} ${pkgsrc}\" &
+			done" &
+		else
+			ssh $port $client "PBULK_CONF=${PBULK_CONF} ${pscan} -c ${master_port_scan} -M ${make} ${extra_pscan_args} ${pkgsrc}" &
+		fi
 	else
 		chroot "$path" ${pscan} -c ${master_port_scan} -M ${make} ${extra_pscan_args} ${pkgsrc} &
 	fi

From 03453d4f3dcffd7eeef3a4f5d8929a5e6c0ce09d Mon Sep 17 00:00:00 2001
From: Jonathan Perkin <jperkin@joyent.com>
Date: Fri, 16 Oct 2015 14:39:10 +0100
Subject: [PATCH 03/28] Support specifying pbulk.conf on the command line.

This allows a single pbulk install to handle multiple builds easily
without having to switch out the main pbulk.conf each time.
---
 pkgtools/pbulk/files/pbulk/scripts/build       |  2 +-
 .../files/pbulk/scripts/build-client-start     |  2 +-
 pkgtools/pbulk/files/pbulk/scripts/bulkbuild   | 16 +++++++++++++++-
 .../files/pbulk/scripts/bulkbuild-rebuild      | 18 ++++++++++++++++--
 .../files/pbulk/scripts/bulkbuild-restart      | 16 +++++++++++++++-
 .../pbulk/files/pbulk/scripts/client-clean     |  2 +-
 .../pbulk/files/pbulk/scripts/client-prepare   |  2 +-
 pkgtools/pbulk/files/pbulk/scripts/pkg-build   |  2 +-
 .../pbulk/files/pbulk/scripts/pkg-up-to-date   |  2 +-
 pkgtools/pbulk/files/pbulk/scripts/pre-build   |  2 +-
 pkgtools/pbulk/files/pbulk/scripts/report      |  2 +-
 pkgtools/pbulk/files/pbulk/scripts/scan        |  2 +-
 .../files/pbulk/scripts/scan-client-start      |  2 +-
 pkgtools/pbulk/files/pbulk/scripts/upload      |  2 +-
 14 files changed, 57 insertions(+), 15 deletions(-)

diff --git a/pkgtools/pbulk/files/pbulk/scripts/build b/pkgtools/pbulk/files/pbulk/scripts/build
index 494e8574bec5..dfb7e5e0cbbc 100755
--- a/pkgtools/pbulk/files/pbulk/scripts/build
+++ b/pkgtools/pbulk/files/pbulk/scripts/build
@@ -30,7 +30,7 @@
 # OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 # SUCH DAMAGE.
 
-. @PBULK_CONFIG@
+. ${PBULK_CONF:-@PBULK_CONFIG@}
 
 set -e
 
diff --git a/pkgtools/pbulk/files/pbulk/scripts/build-client-start b/pkgtools/pbulk/files/pbulk/scripts/build-client-start
index c557044e9507..493fc09abe42 100755
--- a/pkgtools/pbulk/files/pbulk/scripts/build-client-start
+++ b/pkgtools/pbulk/files/pbulk/scripts/build-client-start
@@ -1,7 +1,7 @@
 #!@SH@
 # $NetBSD: build-client-start,v 1.5 2016/12/18 22:59:35 joerg Exp $
 
-. @PBULK_CONFIG@
+. ${PBULK_CONF:-@PBULK_CONFIG@}
 
 set -e
 
diff --git a/pkgtools/pbulk/files/pbulk/scripts/bulkbuild b/pkgtools/pbulk/files/pbulk/scripts/bulkbuild
index 4b0c233b76ba..b4ada70aab86 100755
--- a/pkgtools/pbulk/files/pbulk/scripts/bulkbuild
+++ b/pkgtools/pbulk/files/pbulk/scripts/bulkbuild
@@ -1,7 +1,21 @@
 #!@SH@
 # $NetBSD: bulkbuild,v 1.1.1.1 2007/06/19 19:49:59 joerg Exp $
 
-. @PBULK_CONFIG@
+if [ "$1" = "-h" ]; then
+	echo "usage: $0 [-h] [<config file>]"
+	exit 2
+elif [ -f "$1" ]; then
+	case "$1" in
+	/*)
+		export PBULK_CONF="$1"; shift
+		;;
+	*)
+		export PBULK_CONF="$(pwd)/$1"; shift
+		;;
+	esac
+fi
+
+. ${PBULK_CONF:-@PBULK_CONFIG@}
 
 set -e
 
diff --git a/pkgtools/pbulk/files/pbulk/scripts/bulkbuild-rebuild b/pkgtools/pbulk/files/pbulk/scripts/bulkbuild-rebuild
index 33ac0f39eb14..cbfb18a97205 100755
--- a/pkgtools/pbulk/files/pbulk/scripts/bulkbuild-rebuild
+++ b/pkgtools/pbulk/files/pbulk/scripts/bulkbuild-rebuild
@@ -1,7 +1,21 @@
 #!@SH@
 # $NetBSD: bulkbuild-rebuild,v 1.3 2008/09/16 18:21:30 joerg Exp $
 
-. @PBULK_CONFIG@
+if [ "$1" = "-h" ]; then
+	echo "Usage: $0 [-h] [<config file>] package ..."
+	exit 2
+elif [ -f "$1" ]; then
+	case "$1" in
+	/*)
+		export PBULK_CONF="$1"; shift
+		;;
+	*)
+		export PBULK_CONF="$(pwd)/$1"; shift
+		;;
+	esac
+fi
+
+. ${PBULK_CONF:-@PBULK_CONFIG@}
 
 set -e
 
@@ -33,7 +47,7 @@ build_pkg() {
 }
 
 if [ $# = 0 ]; then
-	echo "Usage: $0 package ..."
+	echo "Usage: $0 [-h] [<config file>] package ..."
 fi
 
 while [ $# != 0 ]; do
diff --git a/pkgtools/pbulk/files/pbulk/scripts/bulkbuild-restart b/pkgtools/pbulk/files/pbulk/scripts/bulkbuild-restart
index 85424890db0b..beb470502bb4 100755
--- a/pkgtools/pbulk/files/pbulk/scripts/bulkbuild-restart
+++ b/pkgtools/pbulk/files/pbulk/scripts/bulkbuild-restart
@@ -1,7 +1,21 @@
 #!@SH@
 # $NetBSD: bulkbuild-restart,v 1.4 2016/12/18 22:38:16 joerg Exp $
 
-. @PBULK_CONFIG@
+if [ "$1" = "-h" ]; then
+	echo "usage: $0 [-h] [<config file>]"
+	exit 2
+elif [ -f "$1" ]; then
+	case "$1" in
+	/*)
+		export PBULK_CONF="$1"; shift
+		;;
+	*)
+		export PBULK_CONF="$(pwd)/$1"; shift
+		;;
+	esac
+fi
+
+. ${PBULK_CONF:-@PBULK_CONFIG@}
 
 set -e
 
diff --git a/pkgtools/pbulk/files/pbulk/scripts/client-clean b/pkgtools/pbulk/files/pbulk/scripts/client-clean
index e94312dc3be0..90048fc9eff4 100755
--- a/pkgtools/pbulk/files/pbulk/scripts/client-clean
+++ b/pkgtools/pbulk/files/pbulk/scripts/client-clean
@@ -1,7 +1,7 @@
 #!@SH@
 # $NetBSD: client-clean,v 1.9 2008/09/16 18:21:30 joerg Exp $
 
-. @PBULK_CONFIG@
+. ${PBULK_CONF:-@PBULK_CONFIG@}
 
 set -e
 
diff --git a/pkgtools/pbulk/files/pbulk/scripts/client-prepare b/pkgtools/pbulk/files/pbulk/scripts/client-prepare
index 75bb0550c41a..52280847c7a7 100755
--- a/pkgtools/pbulk/files/pbulk/scripts/client-prepare
+++ b/pkgtools/pbulk/files/pbulk/scripts/client-prepare
@@ -1,7 +1,7 @@
 #!@SH@
 # $NetBSD: client-prepare,v 1.5 2020/04/04 15:17:48 riastradh Exp $
 
-. @PBULK_CONFIG@
+. ${PBULK_CONF:-@PBULK_CONFIG@}
 
 set -e
 
diff --git a/pkgtools/pbulk/files/pbulk/scripts/pkg-build b/pkgtools/pbulk/files/pbulk/scripts/pkg-build
index f11d4c2e5da2..f64f77083fd8 100755
--- a/pkgtools/pbulk/files/pbulk/scripts/pkg-build
+++ b/pkgtools/pbulk/files/pbulk/scripts/pkg-build
@@ -30,7 +30,7 @@
 # OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 # SUCH DAMAGE.
 
-. @PBULK_CONFIG@
+. ${PBULK_CONF:-@PBULK_CONFIG@}
 
 if [ "${config_version}" != "@PBULK_CONFIG_VERSION@" ]; then
 	echo "Your configuration has version ${config_version}."
diff --git a/pkgtools/pbulk/files/pbulk/scripts/pkg-up-to-date b/pkgtools/pbulk/files/pbulk/scripts/pkg-up-to-date
index 7c476967fcfb..4000bd01ed4d 100755
--- a/pkgtools/pbulk/files/pbulk/scripts/pkg-up-to-date
+++ b/pkgtools/pbulk/files/pbulk/scripts/pkg-up-to-date
@@ -30,7 +30,7 @@
 # OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 # SUCH DAMAGE.
 
-. @PBULK_CONFIG@
+. ${PBULK_CONF:-@PBULK_CONFIG@}
 
 set -e
 
diff --git a/pkgtools/pbulk/files/pbulk/scripts/pre-build b/pkgtools/pbulk/files/pbulk/scripts/pre-build
index 181d277aadf4..d6806ba4cb91 100755
--- a/pkgtools/pbulk/files/pbulk/scripts/pre-build
+++ b/pkgtools/pbulk/files/pbulk/scripts/pre-build
@@ -30,7 +30,7 @@
 # OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 # SUCH DAMAGE.
 
-. @PBULK_CONFIG@
+. ${PBULK_CONF:-@PBULK_CONFIG@}
 
 set -e
 
diff --git a/pkgtools/pbulk/files/pbulk/scripts/report b/pkgtools/pbulk/files/pbulk/scripts/report
index e207018ae89f..30ea3f0d4019 100755
--- a/pkgtools/pbulk/files/pbulk/scripts/report
+++ b/pkgtools/pbulk/files/pbulk/scripts/report
@@ -30,7 +30,7 @@
 # OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 # SUCH DAMAGE.
 
-. @PBULK_CONFIG@
+. ${PBULK_CONF:-@PBULK_CONFIG@}
 
 set -e
 
diff --git a/pkgtools/pbulk/files/pbulk/scripts/scan b/pkgtools/pbulk/files/pbulk/scripts/scan
index d24f0dfc2ea6..028bf7ee0af5 100755
--- a/pkgtools/pbulk/files/pbulk/scripts/scan
+++ b/pkgtools/pbulk/files/pbulk/scripts/scan
@@ -30,7 +30,7 @@
 # OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 # SUCH DAMAGE.
 
-. @PBULK_CONFIG@
+. ${PBULK_CONF:-@PBULK_CONFIG@}
 
 set -e
 
diff --git a/pkgtools/pbulk/files/pbulk/scripts/scan-client-start b/pkgtools/pbulk/files/pbulk/scripts/scan-client-start
index bcf58077b986..9b6acf75c43d 100755
--- a/pkgtools/pbulk/files/pbulk/scripts/scan-client-start
+++ b/pkgtools/pbulk/files/pbulk/scripts/scan-client-start
@@ -1,7 +1,7 @@
 #!@SH@
 # $NetBSD: scan-client-start,v 1.7 2016/12/18 22:59:35 joerg Exp $
 
-. @PBULK_CONFIG@
+. ${PBULK_CONF:-@PBULK_CONFIG@}
 
 set -e
 
diff --git a/pkgtools/pbulk/files/pbulk/scripts/upload b/pkgtools/pbulk/files/pbulk/scripts/upload
index 2a17168f83b1..7dbc8028c0dc 100755
--- a/pkgtools/pbulk/files/pbulk/scripts/upload
+++ b/pkgtools/pbulk/files/pbulk/scripts/upload
@@ -30,7 +30,7 @@
 # OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 # SUCH DAMAGE.
 
-. @PBULK_CONFIG@
+. ${PBULK_CONF:-@PBULK_CONFIG@}
 
 set -e
 

From 0f593531ff44ec63ac90d543715587cb1fe0d5b0 Mon Sep 17 00:00:00 2001
From: Jonathan Perkin <jperkin@joyent.com>
Date: Fri, 16 Oct 2015 16:06:56 +0100
Subject: [PATCH 04/28] Support USER_ADDITIONAL_PKGS.

These are passed in as SUBDIR entries by the top-level Makefile.
---
 pkgtools/pbulk/files/pbulk/pscan/client.c | 24 +++++++++++++++++++++++
 pkgtools/pbulk/files/pbulk/pscan/jobs.c   | 10 ++++++++++
 pkgtools/pbulk/files/pbulk/pscan/pscan.c  | 11 +++++++++++
 pkgtools/pbulk/files/pbulk/pscan/pscan.h  |  1 +
 4 files changed, 46 insertions(+)

diff --git a/pkgtools/pbulk/files/pbulk/pscan/client.c b/pkgtools/pbulk/files/pbulk/pscan/client.c
index 31d613fbd875..59a68956a438 100644
--- a/pkgtools/pbulk/files/pbulk/pscan/client.c
+++ b/pkgtools/pbulk/files/pbulk/pscan/client.c
@@ -80,6 +80,30 @@ find_full_tree_client(int fd, const char *bmake_path)
 		if (cat_len == 0)
 			break;
 
+		/*
+		 * Check for SUBDIR including a '/', if so then it has been
+		 * passed via USER_ADDITIONAL_PKGS from the top-level Makefile,
+		 * and should be added as a package path directly.
+		 */
+		if (memchr(cat, '/', cat_len) != NULL) {
+			if (len_pkgs == allocated_pkgs) {
+				if (allocated_pkgs == 0) {
+					allocated_pkgs = 1024;
+					pkgs = xmalloc(sizeof(*pkgs) *
+							allocated_pkgs);
+				} else {
+					allocated_pkgs *= 2;
+					pkgs = xrealloc(pkgs,
+							sizeof(*pkgs) * allocated_pkgs);
+				}
+			}
+			pkgs[len_pkgs] = xasprintf("%.*s", (int)cat_len, cat);
+			len_pkgs_data += strlen(pkgs[len_pkgs]) + 1;
+			++len_pkgs;
+			cat += cat_len;
+			continue;
+		}
+
 		cat_path = xasprintf("%s/%.*s", pkgsrc_tree, (int)cat_len, cat);
 		buf_orig = buf = read_from_child(cat_path, bmake_path, extract_subdir);
 		free(cat_path);
diff --git a/pkgtools/pbulk/files/pbulk/pscan/jobs.c b/pkgtools/pbulk/files/pbulk/pscan/jobs.c
index fa6284e6ead4..fabce61034d7 100644
--- a/pkgtools/pbulk/files/pbulk/pscan/jobs.c
+++ b/pkgtools/pbulk/files/pbulk/pscan/jobs.c
@@ -278,6 +278,16 @@ add_job(const char *cat, size_t cat_len, const char *dir, size_t dir_len)
 	free(location);
 }
 
+void
+add_job_pkgpath(const char *pkgpath, size_t pkgpath_len)
+{
+	char *location;
+
+	location = xasprintf("%.*s", (int)pkgpath_len, pkgpath);
+	add_job_full(location);
+	free(location);
+}
+
 void
 add_job_full(const char *location)
 {
diff --git a/pkgtools/pbulk/files/pbulk/pscan/pscan.c b/pkgtools/pbulk/files/pbulk/pscan/pscan.c
index 191fb637e940..52e3baddab93 100644
--- a/pkgtools/pbulk/files/pbulk/pscan/pscan.c
+++ b/pkgtools/pbulk/files/pbulk/pscan/pscan.c
@@ -225,6 +225,17 @@ find_full_tree(void)
 		if (cat_len == 0)
 			break;
 
+		/*
+		 * Check for SUBDIR including a '/', if so then it has been
+		 * passed via USER_ADDITIONAL_PKGS from the top-level Makefile,
+		 * and should be added as a package path directly.
+		 */
+		if (memchr(cat, '/', cat_len) != NULL) {
+			add_job_pkgpath(cat, cat_len);
+			cat += cat_len;
+			continue;
+		}
+
 		cat_path = xasprintf("%s/%.*s", pkgsrc_tree, (int)cat_len, cat);
 		buf_orig = buf = read_from_child(cat_path, bmake_path, extract_subdir);
 		free(cat_path);
diff --git a/pkgtools/pbulk/files/pbulk/pscan/pscan.h b/pkgtools/pbulk/files/pbulk/pscan/pscan.h
index dd6f2156da28..da4f7e50f458 100644
--- a/pkgtools/pbulk/files/pbulk/pscan/pscan.h
+++ b/pkgtools/pbulk/files/pbulk/pscan/pscan.h
@@ -56,6 +56,7 @@ void		 client_mode(const char *, const char *);
 void		 master_mode(const char *, const char *);
 
 void		 add_job(const char *, size_t, const char *, size_t);
+void		 add_job_pkgpath(const char *, size_t);
 void		 add_job_full(const char *);
 struct scan_job	*get_job(void);
 void		 process_job(struct scan_job *, enum job_state);

From 1f562d56b1b7096544daa468073c17168c3509ee Mon Sep 17 00:00:00 2001
From: Jonathan Perkin <jperkin@joyent.com>
Date: Fri, 16 Oct 2015 16:31:12 +0100
Subject: [PATCH 05/28] Support optional $client_clean_local script.

This allows local customisations to the client-clean script, for example
our OSX /tmp ramdisk cleanup:

  https://github.com/joyent/pkgbuild/blob/master/scripts/client-clean-osx

which is necessary to keep the ramdisk usage pruned.
---
 pkgtools/pbulk/files/pbulk/pbulk.conf           | 4 ++++
 pkgtools/pbulk/files/pbulk/scripts/client-clean | 5 +++++
 2 files changed, 9 insertions(+)

diff --git a/pkgtools/pbulk/files/pbulk/pbulk.conf b/pkgtools/pbulk/files/pbulk/pbulk.conf
index bff42082a5a3..2c1695e4abdb 100644
--- a/pkgtools/pbulk/files/pbulk/pbulk.conf
+++ b/pkgtools/pbulk/files/pbulk/pbulk.conf
@@ -160,3 +160,7 @@ sync_buildlog=:
 sync_package=default_sync_package
 # PKG_SUFX as set in mk.conf
 pkg_sufx=.tgz
+
+# Optional local override scripts.
+#
+#client_clean_local=/client_clean_local
diff --git a/pkgtools/pbulk/files/pbulk/scripts/client-clean b/pkgtools/pbulk/files/pbulk/scripts/client-clean
index 90048fc9eff4..235c205c63cb 100755
--- a/pkgtools/pbulk/files/pbulk/scripts/client-clean
+++ b/pkgtools/pbulk/files/pbulk/scripts/client-clean
@@ -23,6 +23,11 @@ rm -rf ${cur_destdir}${varbase}/qmail 2> /dev/null || true
 # Note: It is not an error if ${prefix} still exists at this point,
 # because it might be a mount point.
 
+# Source optional cleanup script
+if [ -f "${client_clean_local}" ]; then
+	. ${client_clean_local}
+fi
+
 # Install fresh bootstrap state
 if [ -f "${bootstrapkit}" ]; then
 	${tar} -xzf ${bootstrapkit} -C ${cur_destdir}/

From 80ce299640624fa62377a205dd961021fc523e27 Mon Sep 17 00:00:00 2001
From: Jonathan Perkin <jperkin@joyent.com>
Date: Fri, 16 Oct 2015 16:34:27 +0100
Subject: [PATCH 06/28] Don't recursively mark packages as restricted.

This is useful when e.g. using Oracle Java which cannot be
redistributed.  It should still be possible to install packages which
depend upon it after the user has installed the restricted package
manually first.
---
 pkgtools/pbulk/files/pbulk/scripts/compute-packages.awk | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pkgtools/pbulk/files/pbulk/scripts/compute-packages.awk b/pkgtools/pbulk/files/pbulk/scripts/compute-packages.awk
index d9de4becb89c..63bf9dfe812c 100755
--- a/pkgtools/pbulk/files/pbulk/scripts/compute-packages.awk
+++ b/pkgtools/pbulk/files/pbulk/scripts/compute-packages.awk
@@ -59,7 +59,7 @@ BEGIN {
 			status[cur] = substr($0, 14)
 
 		if ($0 ~ "^NO_BIN_ON_FTP=.")
-			initial_restricted[cur] = 1
+			restricted[cur] = 1
 
 		if ($0 ~ "^DEPENDS=")
 			depends[cur] = substr($0, 9)

From 99b2da7b74eb8d7c912a8391e53bb6e8ffbb4c7f Mon Sep 17 00:00:00 2001
From: Jonathan Perkin <jperkin@joyent.com>
Date: Fri, 16 Oct 2015 16:40:46 +0100
Subject: [PATCH 07/28] Increase maximum top broken package count to 25.

---
 pkgtools/pbulk/files/pbulk/scripts/create-report-html.awk | 2 +-
 pkgtools/pbulk/files/pbulk/scripts/create-report-txt.awk  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/pkgtools/pbulk/files/pbulk/scripts/create-report-html.awk b/pkgtools/pbulk/files/pbulk/scripts/create-report-html.awk
index 460b140ce8d5..62fe863b3d03 100755
--- a/pkgtools/pbulk/files/pbulk/scripts/create-report-html.awk
+++ b/pkgtools/pbulk/files/pbulk/scripts/create-report-html.awk
@@ -310,7 +310,7 @@ BEGIN {
 		print "      </thead>" > html_report
 		print "      <tbody>" > html_report
 
-		for (i = 0; i < 10 && sorted_top_count[i] != ""; ++i) {
+		for (i = 0; i < 25 && sorted_top_count[i] != ""; ++i) {
 			pkg = top_count[sorted_top_count[i]]
 			print_failed(pkg)
 		}
diff --git a/pkgtools/pbulk/files/pbulk/scripts/create-report-txt.awk b/pkgtools/pbulk/files/pbulk/scripts/create-report-txt.awk
index fccc1b20ca09..465ccb12b0be 100755
--- a/pkgtools/pbulk/files/pbulk/scripts/create-report-txt.awk
+++ b/pkgtools/pbulk/files/pbulk/scripts/create-report-txt.awk
@@ -153,7 +153,7 @@ BEGIN {
 		print "" > txt_report
 		print "Package                               Breaks Maintainer" > txt_report
 		print "-------------------------------------------------------------------------" > txt_report
-		for (i = 0; i < 10 && sorted_top_count[i] != ""; ++i) {
+		for (i = 0; i < 25 && sorted_top_count[i] != ""; ++i) {
 			loc = top_count[sorted_top_count[i]]
 			printf "%- 37s % 6d %s\n", loc, broken_location[loc],
 			    maintainer[pkg_location[loc]] > txt_report

From 3eb5e5dd86d8aea60c939528eda90bb3b62a073e Mon Sep 17 00:00:00 2001
From: Jonathan Perkin <jperkin@joyent.com>
Date: Fri, 16 Oct 2015 16:42:57 +0100
Subject: [PATCH 08/28] Replace hardcoded ".tgz" with $pkg_sufx

---
 pkgtools/pbulk/files/pbulk/scripts/compute-packages.awk | 3 ++-
 pkgtools/pbulk/files/pbulk/scripts/pkg-up-to-date       | 4 ++--
 pkgtools/pbulk/files/pbulk/scripts/upload               | 2 +-
 3 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/pkgtools/pbulk/files/pbulk/scripts/compute-packages.awk b/pkgtools/pbulk/files/pbulk/scripts/compute-packages.awk
index 63bf9dfe812c..deb5dba744ce 100755
--- a/pkgtools/pbulk/files/pbulk/scripts/compute-packages.awk
+++ b/pkgtools/pbulk/files/pbulk/scripts/compute-packages.awk
@@ -43,6 +43,7 @@ function mark_restricted(PKG, dep, depend_list) {
 
 BEGIN {
 	meta_dir = ARGV[1]
+	pkg_sufx = ARGV[2]
 	success_file = meta_dir "/success"
 	presolve_file = meta_dir "/presolve"
 
@@ -90,7 +91,7 @@ BEGIN {
 				print "+ " cat "/"
 				printed_cats[cat] = cat
 			}
-			print "+ " cat "/" pkg ".tgz"
+			print "+ " cat "/" pkg pkg_sufx
 		}
 	}
 	close(success_file)
diff --git a/pkgtools/pbulk/files/pbulk/scripts/pkg-up-to-date b/pkgtools/pbulk/files/pbulk/scripts/pkg-up-to-date
index 4000bd01ed4d..9e96a8326294 100755
--- a/pkgtools/pbulk/files/pbulk/scripts/pkg-up-to-date
+++ b/pkgtools/pbulk/files/pbulk/scripts/pkg-up-to-date
@@ -40,7 +40,7 @@ if [ "${config_version}" != "@PBULK_CONFIG_VERSION@" ]; then
 	exit 1
 fi
 
-pkg="${packages}/All/$1.tgz"
+pkg="${packages}/All/${1}${pkg_sufx}"
 [ -f ${pkg} ]
 
 ${pkg_info} -qb ${pkg} | sed 's/:/ /' | while read file file_id; do
@@ -79,7 +79,7 @@ ${pkg_info} -qN ${pkg} | while read dep; do
 	[Yy][Ee][Ss])
 		;;
 	*)
-		[ "${packages}/All/${dep}.tgz" -ot "${pkg}" ]
+		[ "${packages}/All/${dep}${pkg_sufx}" -ot "${pkg}" ]
 		;;
 	esac
 done
diff --git a/pkgtools/pbulk/files/pbulk/scripts/upload b/pkgtools/pbulk/files/pbulk/scripts/upload
index 7dbc8028c0dc..5131a9d386aa 100755
--- a/pkgtools/pbulk/files/pbulk/scripts/upload
+++ b/pkgtools/pbulk/files/pbulk/scripts/upload
@@ -48,6 +48,6 @@ cd ${packages}
 	    echo "+ SHA512.bz2"
 	echo "+ All/pkg_summary.bz2"
 	echo "+ All/pkg_summary.gz"
-	${packages_script} ${loc}
+	${packages_script} ${loc} ${pkg_sufx}
 	echo "- *"
 } | LC_ALL=C sort | ${rsync} --exclude-from=- ${pkg_rsync_args} . ${pkg_rsync_target}

From f7fa8f85e74ce04411c2b6478e75ce79e02faa77 Mon Sep 17 00:00:00 2001
From: Jonathan Perkin <jperkin@joyent.com>
Date: Fri, 16 Oct 2015 16:49:58 +0100
Subject: [PATCH 09/28] Move OS variable generation to the first build.

There's no guarantee that the pbulk master is running the same OS
version that the pbulk clients are.
---
 pkgtools/pbulk/files/pbulk/scripts/pkg-build | 32 +++++++++++++++++++
 pkgtools/pbulk/files/pbulk/scripts/pre-build | 33 ++------------------
 2 files changed, 35 insertions(+), 30 deletions(-)

diff --git a/pkgtools/pbulk/files/pbulk/scripts/pkg-build b/pkgtools/pbulk/files/pbulk/scripts/pkg-build
index f64f77083fd8..dddf132c2812 100755
--- a/pkgtools/pbulk/files/pbulk/scripts/pkg-build
+++ b/pkgtools/pbulk/files/pbulk/scripts/pkg-build
@@ -135,6 +135,38 @@ if [ `@ID@ -u` -ne 0 ]; then
 	run_install=run_direct
 fi
 
+# Attempt to move the pre-build status file into place.  If we're successful
+# we are the first build and we can now log the remaining details.
+ret=`mv ${loc}/status.pre-build ${loc}/status 2>/dev/null; echo $?`
+if [ ${ret} -eq 0 ]; then
+	@PREFIX@/libexec/pbulk/client-clean
+	opsys=`cd ${pkgsrc}/pkgtools/pkg_install && ${make} show-var VARNAME=OPSYS`
+	opver=`cd ${pkgsrc}/pkgtools/pkg_install && ${make} show-var VARNAME=OS_VERSION`
+	# Check if it is an OS with variants where we would rather use the
+	# variant name and version rather than common name and version
+	case "$opsys" in
+	SunOS | Cygwin | SCO_SV)
+		opvar=`cd ${pkgsrc}/pkgtools/pkg_install && ${make} show-var VARNAME=OS_VARIANT`
+		case "$opsys" in
+		Cygwin | SCO_SV)
+			opsys=$opvar
+			;;
+		SunOS)
+			if [ -n "$opvar" ]; then
+				opsys=$opvar
+				opver=`cd ${pkgsrc}/pkgtools/pkg_install && ${make} show-var VARNAME=LOWER_VARIANT_VERSION`
+			fi
+			;;
+		esac
+		;;
+	esac
+	platform=`cd ${pkgsrc}/pkgtools/pkg_install && ${make} show-var VARNAME=MACHINE_ARCH`
+	compiler=`cd ${pkgsrc}/pkgtools/pkg_install && ${make} show-var VARNAME=PKGSRC_COMPILER`
+
+	echo "COMPILER=${compiler}" >> ${loc}/status
+	echo "PLATFORM=${opsys} ${opver}/${platform}" >> ${loc}/status
+fi
+
 ${pkg_up_to_date_script} ${pkgname} ${dependencies} && exit 0
 
 set -e
diff --git a/pkgtools/pbulk/files/pbulk/scripts/pre-build b/pkgtools/pbulk/files/pbulk/scripts/pre-build
index d6806ba4cb91..52f37809a125 100755
--- a/pkgtools/pbulk/files/pbulk/scripts/pre-build
+++ b/pkgtools/pbulk/files/pbulk/scripts/pre-build
@@ -101,35 +101,8 @@ case "${master_mode}" in
 	;;
 esac
 
-# Log common settings...
-opsys=`cd ${pkgsrc}/pkgtools/pkg_install && ${make} show-var VARNAME=OPSYS`
-opver=`cd ${pkgsrc}/pkgtools/pkg_install && ${make} show-var VARNAME=OS_VERSION`
-# Check if it is an OS with variants where we would rather use the
-# variant name and version rather than common name and version
-case "$opsys" in
-	SunOS | Cygwin | SCO_SV)
-		opvar=`cd ${pkgsrc}/pkgtools/pkg_install && ${make} show-var VARNAME=OS_VARIANT`
-	case "$opsys" in
-		Cygwin | SCO_SV)
-			opsys=$opvar
-		;;
-		SunOS)
-			if [ -n "$opvar" ]; then
-			opsys=$opvar
-			opver=`cd ${pkgsrc}/pkgtools/pkg_install && ${make} show-var VARNAME=LOWER_VARIANT_VERSION`
-			fi
-		;;
-	esac
-	;;
-esac
-
-platform=`cd ${pkgsrc}/pkgtools/pkg_install && ${make} show-var VARNAME=MACHINE_ARCH`
-compiler=`cd ${pkgsrc}/pkgtools/pkg_install && ${make} show-var VARNAME=PKGSRC_COMPILER`
-
-echo "PLATFORM=${opsys} ${opver}/${platform}" > ${loc}/status
-echo "COMPILER=${compiler}" >> ${loc}/status
-
+# Start status file, it will be moved into place by the first build.
 date '+BUILD_START_ISO=%Y-%m-%d %H:%M
 BUILD_START_DIR=%Y%m%d.%H%M
-BUILD_START_EPOCHE=%s' >> ${loc}/status
-echo "BASE_URL=${base_url}" >> ${loc}/status
+BUILD_START_EPOCHE=%s' > ${loc}/status.pre-build
+echo "BASE_URL=${base_url}" >> ${loc}/status.pre-build

From ff951093ba2d1ef8c2aa7b0281cc014e5ac057f1 Mon Sep 17 00:00:00 2001
From: Jonathan Perkin <jperkin@joyent.com>
Date: Fri, 16 Oct 2015 16:54:31 +0100
Subject: [PATCH 10/28] Ensure each build starts with clean passwd files.

Skipped on Darwin as it has a centralised OpenDirectory database,
however finding a way to do this on Darwin would be greatly beneficial.
---
 pkgtools/pbulk/files/pbulk/scripts/pkg-build | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/pkgtools/pbulk/files/pbulk/scripts/pkg-build b/pkgtools/pbulk/files/pbulk/scripts/pkg-build
index dddf132c2812..1aa8af263024 100755
--- a/pkgtools/pbulk/files/pbulk/scripts/pkg-build
+++ b/pkgtools/pbulk/files/pbulk/scripts/pkg-build
@@ -169,6 +169,17 @@ fi
 
 ${pkg_up_to_date_script} ${pkgname} ${dependencies} && exit 0
 
+# Ensure we start with a clean passwd/group.
+if [ `uname` != "Darwin" ]; then
+	for file in group passwd shadow user_attr; do
+		if [ -f ${loc}/${file} ]; then
+			cp -p ${loc}/${file} /etc/${file}
+		elif [ -f /etc/${file} ]; then
+			cp -p /etc/${file} ${loc}/${file}
+		fi
+	done
+fi
+
 set -e
 
 @PREFIX@/libexec/pbulk/client-clean

From 2d6805f84dc4b3cb2bd5f2905ed5eed2b7c485f5 Mon Sep 17 00:00:00 2001
From: Jonathan Perkin <jperkin@joyent.com>
Date: Fri, 16 Oct 2015 16:56:17 +0100
Subject: [PATCH 11/28] Support uploading reports to Manta.

Assumes pbulk.conf is populated with the appropriate variables and that
nodejs and the 'manta' npm package is installed under $MANTA_PATH.
---
 pkgtools/pbulk/files/pbulk/scripts/report | 17 ++++++++++++++++-
 1 file changed, 16 insertions(+), 1 deletion(-)

diff --git a/pkgtools/pbulk/files/pbulk/scripts/report b/pkgtools/pbulk/files/pbulk/scripts/report
index 30ea3f0d4019..9061b0a253a8 100755
--- a/pkgtools/pbulk/files/pbulk/scripts/report
+++ b/pkgtools/pbulk/files/pbulk/scripts/report
@@ -107,4 +107,19 @@ echo "Uploading report..."
 		[ -f "$file" ] && echo "+ $file"
 	done
 	echo "- *"
-} | ${rsync} --exclude-from=- ${report_rsync_args} . ${report_rsync_target}/${build_start_dir}
+} | {
+	if [ -n "${report_manta_target}" ]; then
+		# Variables sourced from pbulk.conf
+		export MANTA_USER MANTA_KEY_ID MANTA_URL MANTA_SUBUSER
+		PATH=${MANTA_PATH}:${PATH}
+		# Output directory only needs to not exist or be empty.
+		tmpdir="${bulklog}/pbulk-rsync-dir"
+		tmptar="${bulklog}/pbulk-rsync.tar"
+		rm -rf ${tmpdir} ${tmptar}
+		${rsync} -rn --exclude-from=- --out-format='%n' . ${tmpdir} \
+		    | grep -v '/$' | @TAR@ -cf ${tmptar} -T -
+		muntar -f ${tmptar} ${report_manta_target}/${build_start_dir}
+	else
+		${rsync} --exclude-from=- ${report_rsync_args} . ${report_rsync_target}/${build_start_dir}
+	fi
+}

From 49281bd9c11901d80c3d7df3bc870a70a2d387e9 Mon Sep 17 00:00:00 2001
From: Jonathan Perkin <jperkin@joyent.com>
Date: Fri, 16 Oct 2015 17:00:23 +0100
Subject: [PATCH 12/28] Generate checksum and pkginfo at package build time.

This saves a decent amount of time at the end of a pbulk run (around 20
minutes) by avoiding a single-threaded operation to generate them.

XXX: not currently handled by custom $sync_package scripts
---
 pkgtools/pbulk/files/pbulk/scripts/build     | 10 +++++-----
 pkgtools/pbulk/files/pbulk/scripts/pkg-build | 18 ++++++++++++++++++
 2 files changed, 23 insertions(+), 5 deletions(-)

diff --git a/pkgtools/pbulk/files/pbulk/scripts/build b/pkgtools/pbulk/files/pbulk/scripts/build
index dfb7e5e0cbbc..0753992fd5f8 100755
--- a/pkgtools/pbulk/files/pbulk/scripts/build
+++ b/pkgtools/pbulk/files/pbulk/scripts/build
@@ -61,7 +61,7 @@ BUILD_END_EPOCHE=%s' >> ${loc}/status
 
 echo "Building pkg_summary..."
 cd ${packages}/All
-sed 's/$/.tgz/' < ${loc}/success | sort | xargs ${pkg_info} -X | ${gzip} -c > pkg_summary.gz
+sed -e 's,^,../pkginfo/,' -e 's/$/.pkginfo/' < ${loc}/success | sort | xargs cat 2>/dev/null | ${gzip} -c > pkg_summary.gz
 ${gzip} -dc < pkg_summary.gz | ${bzip2} -c > pkg_summary.bz2
 
 if [ "${checksum_packages}" != "no" ] && \
@@ -69,8 +69,8 @@ if [ "${checksum_packages}" != "no" ] && \
 	echo "Building SHA512..."
 	cd ${packages}
 	{
-		echo "All/pkg_summary.bz2"
-		echo "All/pkg_summary.gz"
-		sed 's|^\(.*\)$|All/\1.tgz|' < ${loc}/success
-	} | sort | xargs ${digest} SHA512 | ${bzip2} -c > SHA512.bz2
+		${digest} SHA512 All/pkg_summary.bz2
+		${digest} SHA512 All/pkg_summary.gz
+		sed -e 's,^,checksums/,' -e 's/$/.sha512/' < ${loc}/success | xargs cat 2>/dev/null
+	} | sort | ${bzip2} -c > SHA512.bz2
 fi
diff --git a/pkgtools/pbulk/files/pbulk/scripts/pkg-build b/pkgtools/pbulk/files/pbulk/scripts/pkg-build
index 1aa8af263024..43720cfd0619 100755
--- a/pkgtools/pbulk/files/pbulk/scripts/pkg-build
+++ b/pkgtools/pbulk/files/pbulk/scripts/pkg-build
@@ -113,6 +113,10 @@ while read build_info_line; do
 		;;
 	USERGROUP_PHASE=*)
 		usergroup_phase=${build_info_line#USERGROUP_PHASE=}
+		;;
+	NO_BIN_ON_FTP=*)
+		restricted=${build_info_line#NO_BIN_ON_FTP=}
+		;;
 	esac
 done
 
@@ -262,5 +266,19 @@ ${sync_package} ${pkgfile} ${pkgname} "${categories}" || cleanup
 # Clean build area
 ${make} clean > ${bulklog}/${pkgname}/clean.log 2>&1 || true
 
+# Save a SHA512 sum of the package (if not restricted), saves doing all of
+# them single-threaded at the end of a bulk build and avoids re-computing for
+# packages we have already built.
+if [ -z "${restricted}" ]; then
+	mkdir -p ${packages}/checksums ${packages}/pkginfo
+	(
+		cd ${packages}
+		${digest} SHA512 All/${pkgname}${pkg_sufx} \
+		    > ${packages}/checksums/${pkgname}.sha512 || cleanup
+		${pkg_info} -X All/${pkgname}${pkg_sufx} \
+		    > ${packages}/pkginfo/${pkgname}.pkginfo || cleanup
+	)
+fi
+
 # Cleanup build logs on success
 rm -R ${bulklog}/${pkgname}

From 450017f89f5fc316c46177ea0fafa25e3a0c4984 Mon Sep 17 00:00:00 2001
From: Jonathan Perkin <jperkin@joyent.com>
Date: Fri, 16 Oct 2015 17:10:12 +0100
Subject: [PATCH 13/28] Save package build time for performance analysis.

---
 pkgtools/pbulk/files/pbulk/scripts/pkg-build | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/pkgtools/pbulk/files/pbulk/scripts/pkg-build b/pkgtools/pbulk/files/pbulk/scripts/pkg-build
index 43720cfd0619..2032a971049c 100755
--- a/pkgtools/pbulk/files/pbulk/scripts/pkg-build
+++ b/pkgtools/pbulk/files/pbulk/scripts/pkg-build
@@ -43,7 +43,15 @@ default_sync_package() {
 	cp "$1" "${packages}/All/${pkgname}${pkg_sufx}"
 }
 
+logpkgtime() {
+	stop_time=`date '+%s'`
+	build_time=`expr ${stop_time} - ${start_time}`
+	mkdir -p ${packages}/pkgtimes
+	echo "${build_time}" >${packages}/pkgtimes/${pkgname}.pkgtime
+}
+
 cleanup() {
+	logpkgtime
 	if [ "$keep_wrkdir" = yes ]; then
 		${make} pbulk-save-wrkdir INTO=${bulklog}/${pkgname}/wrkdir.tar.gz TAR="${tar}"
 	fi
@@ -220,6 +228,9 @@ else
 	cur_pkgdb="${pkgdb}"
 fi
 
+# Save package start time
+start_time=`date '+%s'`
+
 # Go to target directory
 cd ${pkgsrc}/${pkgdir}
 # Clean build area, just in case
@@ -266,6 +277,9 @@ ${sync_package} ${pkgfile} ${pkgname} "${categories}" || cleanup
 # Clean build area
 ${make} clean > ${bulklog}/${pkgname}/clean.log 2>&1 || true
 
+# Save build runtime
+logpkgtime
+
 # Save a SHA512 sum of the package (if not restricted), saves doing all of
 # them single-threaded at the end of a bulk build and avoids re-computing for
 # packages we have already built.

From 3ea59eed6a2dd9786d1e48200c7858e3fac55889 Mon Sep 17 00:00:00 2001
From: Jonathan Perkin <jperkin@joyent.com>
Date: Fri, 16 Oct 2015 17:12:22 +0100
Subject: [PATCH 14/28] Add support for $local_bulklog

This is a directory where the bulklog files are written to temporarily
before either being moved to $bulklog at the end of a failed build or
removed.

This avoids failures when writing to an NFS-backed $bulklog (seen most
commonly with 'gmake: write error'), and also improves performance.
---
 pkgtools/pbulk/files/pbulk/scripts/pkg-build | 52 +++++++++++++-------
 1 file changed, 33 insertions(+), 19 deletions(-)

diff --git a/pkgtools/pbulk/files/pbulk/scripts/pkg-build b/pkgtools/pbulk/files/pbulk/scripts/pkg-build
index 2032a971049c..7208dccd9be2 100755
--- a/pkgtools/pbulk/files/pbulk/scripts/pkg-build
+++ b/pkgtools/pbulk/files/pbulk/scripts/pkg-build
@@ -52,6 +52,10 @@ logpkgtime() {
 
 cleanup() {
 	logpkgtime
+	if [ -n "${local_bulklog}" ]; then
+		mkdir -p ${bulklog}
+		mv ${logpkgdir} ${bulklog}/${pkgname}
+	fi
 	if [ "$keep_wrkdir" = yes ]; then
 		${make} pbulk-save-wrkdir INTO=${bulklog}/${pkgname}/wrkdir.tar.gz TAR="${tar}"
 	fi
@@ -79,7 +83,7 @@ run_make() {
 		BATCH=1 \
 		DEPENDS_TARGET=/nonexistent \
 		${MAKE_FLAGS} \
-		WRKLOG=${bulklog}/${pkgname}/work.log
+		WRKLOG=${logpkgdir}/work.log
 }
 
 run_usergroup() {
@@ -133,6 +137,15 @@ if [ "$use_unprivileged_checksum" != "no" ]; then
 else
 	run_checksum=run_direct
 fi
+
+# Sometimes writing logs to NFS can cause issues, support writing to a local
+# bulklog directory before moving to NFS at the end.
+if [ -n "${local_bulklog}" ]; then
+	logpkgdir="${local_bulklog}/${pkgname}"
+else
+	logpkgdir="${bulklog}/${pkgname}"
+fi
+
 if [ "$use_destdir" = user-destdir ]; then
 	run_build=run_su
 	run_install=run_su
@@ -197,11 +210,12 @@ set -e
 @PREFIX@/libexec/pbulk/client-clean
 
 # Create the output directory and clean it up
-mkdir -p ${bulklog}/${pkgname}
-rm -f ${bulklog}/${pkgname}/*
+rm -rf ${bulklog}/${pkgname}
+mkdir -p ${logpkgdir}
+rm -f ${logpkgdir}/*
 
-touch ${bulklog}/${pkgname}/work.log
-@CHOWN@ ${unprivileged_user} ${bulklog}/${pkgname}/work.log
+touch ${logpkgdir}/work.log
+@CHOWN@ ${unprivileged_user} ${logpkgdir}/work.log
 
 pkg_add_normal() {
 	PKG_PATH=${packages}/All ${pkg_add} -K ${cur_pkgdb} "$@"
@@ -234,20 +248,20 @@ start_time=`date '+%s'`
 # Go to target directory
 cd ${pkgsrc}/${pkgdir}
 # Clean build area, just in case
-${make} clean > ${bulklog}/${pkgname}/pre-clean.log 2>&1
+${make} clean > ${logpkgdir}/pre-clean.log 2>&1 || cleanup
 # Install all dependencies the package said it would need
 if [ ! -z "$dependencies" ]; then
-	${pkg_add_cmd} $dependencies > ${bulklog}/${pkgname}/depends.log 2>&1 || cleanup
+	${pkg_add_cmd} $dependencies > ${logpkgdir}/depends.log 2>&1 || cleanup
 fi
 # Build package, create a separate log file for each major phase
-run_make ${run_checksum} checksum  > ${bulklog}/${pkgname}/checksum.log 2>&1 || cleanup
-run_usergroup configure > ${bulklog}/${pkgname}/configure.log 2>&1 || cleanup
-run_make ${run_build} configure >> ${bulklog}/${pkgname}/configure.log 2>&1 || cleanup
-run_usergroup build> ${bulklog}/${pkgname}/build.log 2>&1 || cleanup
-run_make ${run_build} all >> ${bulklog}/${pkgname}/build.log 2>&1 || cleanup
-run_usergroup install > ${bulklog}/${pkgname}/install.log 2>&1 || cleanup
-run_make ${run_install} stage-install >> ${bulklog}/${pkgname}/install.log 2>&1 || cleanup
-run_make run_direct stage-package-create > ${bulklog}/${pkgname}/package.log 2>&1 || cleanup
+run_make ${run_checksum} checksum > ${logpkgdir}/checksum.log 2>&1 || cleanup
+run_usergroup configure > ${logpkgdir}/configure.log 2>&1 || cleanup
+run_make ${run_build} configure >> ${logpkgdir}/configure.log 2>&1 || cleanup
+run_usergroup build> ${logpkgdir}/build.log 2>&1 || cleanup
+run_make ${run_build} all >> ${logpkgdir}/build.log 2>&1 || cleanup
+run_usergroup install > ${logpkgdir}/install.log 2>&1 || cleanup
+run_make ${run_install} stage-install >> ${logpkgdir}/install.log 2>&1 || cleanup
+run_make run_direct stage-package-create > ${logpkgdir}/package.log 2>&1 || cleanup
 
 pkgfile=$(run_make run_direct show-var VARNAME=STAGE_PKGFILE)
 
@@ -255,7 +269,7 @@ pkgfile=$(run_make run_direct show-var VARNAME=STAGE_PKGFILE)
 # potential bootstrap packages as they might already be installed.
 if [ -z "${is_bootstrap}" ]; then
 	if ! ${pkg_add_cmd} ${pkgfile} \
-	    >> ${bulklog}/${pkgname}/package.log 2>&1; then
+	    >> ${logpkgdir}/package.log 2>&1; then
 		cleanup
 	fi
 fi
@@ -265,7 +279,7 @@ fi
 # part of the bootstrap, those have the preserve flag set.
 if [ "$cross_compile" = "no" ] && \
    [ -z "${is_bootstrap}" ]; then
-	${pkg_delete} -K ${cur_pkgdb} ${pkgname} > ${bulklog}/${pkgname}/deinstall.log 2>&1 || cleanup
+	${pkg_delete} -K ${cur_pkgdb} ${pkgname} > ${logpkgdir}/deinstall.log 2>&1 || cleanup
 fi
 
 # Comment the following out if you want to test all deinstall scripts.
@@ -275,7 +289,7 @@ fi
 ${sync_package} ${pkgfile} ${pkgname} "${categories}" || cleanup
 
 # Clean build area
-${make} clean > ${bulklog}/${pkgname}/clean.log 2>&1 || true
+${make} clean > ${logpkgdir}/clean.log 2>&1 || cleanup
 
 # Save build runtime
 logpkgtime
@@ -295,4 +309,4 @@ if [ -z "${restricted}" ]; then
 fi
 
 # Cleanup build logs on success
-rm -R ${bulklog}/${pkgname}
+rm -R ${logpkgdir}

From 7ba3ada1b1301de0c3a84e8d2fb2c9d53dfadc8f Mon Sep 17 00:00:00 2001
From: Jonathan Perkin <jperkin@joyent.com>
Date: Fri, 16 Oct 2015 17:14:31 +0100
Subject: [PATCH 15/28] Enable pkg-build script logging.

---
 pkgtools/pbulk/files/pbulk/scripts/pkg-build | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/pkgtools/pbulk/files/pbulk/scripts/pkg-build b/pkgtools/pbulk/files/pbulk/scripts/pkg-build
index 7208dccd9be2..efa9ad4d1e36 100755
--- a/pkgtools/pbulk/files/pbulk/scripts/pkg-build
+++ b/pkgtools/pbulk/files/pbulk/scripts/pkg-build
@@ -214,6 +214,11 @@ rm -rf ${bulklog}/${pkgname}
 mkdir -p ${logpkgdir}
 rm -f ${logpkgdir}/*
 
+# Ensure we log any failures from this script itself, otherwise it is very
+# difficult to debug.
+exec >${logpkgdir}/pkg-build.log 2>&1
+set -x
+
 touch ${logpkgdir}/work.log
 @CHOWN@ ${unprivileged_user} ${logpkgdir}/work.log
 

From c8e9d0e344bcdc1f79587bca9887eadc42dd2f6b Mon Sep 17 00:00:00 2001
From: Jonathan Perkin <jperkin@joyent.com>
Date: Fri, 16 Oct 2015 18:04:58 +0100
Subject: [PATCH 16/28] Pull in xz for pkg_summary and the machine report.

---
 pkgtools/pbulk/Makefile                                  | 4 ++--
 pkgtools/pbulk/files/pbulk/pbulk.conf                    | 1 +
 pkgtools/pbulk/files/pbulk/scripts/build                 | 2 ++
 pkgtools/pbulk/files/pbulk/scripts/create-report-txt.awk | 2 +-
 pkgtools/pbulk/files/pbulk/scripts/report                | 4 ++--
 pkgtools/pbulk/files/pbulk/scripts/upload                | 1 +
 6 files changed, 9 insertions(+), 5 deletions(-)

diff --git a/pkgtools/pbulk/Makefile b/pkgtools/pbulk/Makefile
index 68e4d6b52c45..6f2ad9d4af5d 100644
--- a/pkgtools/pbulk/Makefile
+++ b/pkgtools/pbulk/Makefile
@@ -6,7 +6,7 @@ COMMENT=	Modular bulk build framework
 .include "../../pkgtools/pbulk/Makefile.common"
 
 USE_TOOLS+=	awk:run bzip2:run digest:run gzip:run make:run \
-		mail:run sed:run tar:run
+		mail:run sed:run tar:run xz:run
 DEPENDS+=	rsync-[0-9]*:../../net/rsync
 DEPENDS+=	pbulk-base>=0.38:../../pkgtools/pbulk-base
 
@@ -34,7 +34,7 @@ SUBST_FILES.tools=	pbulk.conf scripts/build scripts/build-client-start \
 SUBST_VARS.tools=	AWK BZIP2 CHOWN DIGEST GZIP_CMD ID MAIL_CMD NEATO \
 			PBULK_CONFIG PBULK_CONFIG_VERSION PKG_ADD_CMD \
 			PKG_ADMIN_CMD PKG_DELETE_CMD PKG_INFO_CMD PREFIX \
-			SED SH TAR TARGET_MAKE
+			SED SH TAR TARGET_MAKE XZ
 
 CONF_FILES+=	share/examples/pbulk/pbulk.conf ${PKG_SYSCONFDIR}/pbulk.conf
 
diff --git a/pkgtools/pbulk/files/pbulk/pbulk.conf b/pkgtools/pbulk/files/pbulk/pbulk.conf
index 2c1695e4abdb..5a3059057cf1 100644
--- a/pkgtools/pbulk/files/pbulk/pbulk.conf
+++ b/pkgtools/pbulk/files/pbulk/pbulk.conf
@@ -119,6 +119,7 @@ neato=@NEATO@
 rsync=@PREFIX@/bin/rsync
 sed=@SED@
 tar=@TAR@
+xz=@XZ@
 
 loc=${bulklog}/meta
 
diff --git a/pkgtools/pbulk/files/pbulk/scripts/build b/pkgtools/pbulk/files/pbulk/scripts/build
index 0753992fd5f8..b3ebf0000853 100755
--- a/pkgtools/pbulk/files/pbulk/scripts/build
+++ b/pkgtools/pbulk/files/pbulk/scripts/build
@@ -63,6 +63,7 @@ echo "Building pkg_summary..."
 cd ${packages}/All
 sed -e 's,^,../pkginfo/,' -e 's/$/.pkginfo/' < ${loc}/success | sort | xargs cat 2>/dev/null | ${gzip} -c > pkg_summary.gz
 ${gzip} -dc < pkg_summary.gz | ${bzip2} -c > pkg_summary.bz2
+${gzip} -dc < pkg_summary.gz | ${xz} -zc > pkg_summary.xz
 
 if [ "${checksum_packages}" != "no" ] && \
    [ "${checksum_packages}" != "NO" ]; then
@@ -71,6 +72,7 @@ if [ "${checksum_packages}" != "no" ] && \
 	{
 		${digest} SHA512 All/pkg_summary.bz2
 		${digest} SHA512 All/pkg_summary.gz
+		${digest} SHA512 All/pkg_summary.xz
 		sed -e 's,^,checksums/,' -e 's/$/.sha512/' < ${loc}/success | xargs cat 2>/dev/null
 	} | sort | ${bzip2} -c > SHA512.bz2
 fi
diff --git a/pkgtools/pbulk/files/pbulk/scripts/create-report-txt.awk b/pkgtools/pbulk/files/pbulk/scripts/create-report-txt.awk
index 465ccb12b0be..5e2e48775aab 100755
--- a/pkgtools/pbulk/files/pbulk/scripts/create-report-txt.awk
+++ b/pkgtools/pbulk/files/pbulk/scripts/create-report-txt.awk
@@ -129,7 +129,7 @@ BEGIN {
 	print "" > txt_report
 	report_base_url = pkgsrc_base_url "/" pkgsrc_build_start_dir
 	print "Full report: " report_base_url "/meta/report.html" > txt_report
-	print "Machine readable version: " report_base_url "/meta/report.bz2" > txt_report
+	print "Machine readable version: " report_base_url "/meta/report.xz" > txt_report
 	print "" > txt_report
 	all_pkgs = pkgs_done + pkgs_failed + pkgs_prefailed + pkgs_indirect_failed + pkgs_indirect_prefailed
 	printf "Total number of packages:      %5d\n", all_pkgs > txt_report
diff --git a/pkgtools/pbulk/files/pbulk/scripts/report b/pkgtools/pbulk/files/pbulk/scripts/report
index 9061b0a253a8..498c3135a920 100755
--- a/pkgtools/pbulk/files/pbulk/scripts/report
+++ b/pkgtools/pbulk/files/pbulk/scripts/report
@@ -42,7 +42,7 @@ fi
 
 echo "Build reports..."
 ${report_script} ${loc}
-${bzip2} -zc ${loc}/report > ${loc}/report.bz2
+${xz} -zc ${loc}/report > ${loc}/report.xz
 ${report_html_script} ${loc} ${bulklog}
 ${report_txt_script} ${loc}
 
@@ -79,11 +79,11 @@ cat ${loc}/report.txt | ${mail} -s "${report_subject_prefix} ${platform} ${build
 cd ${bulklog}
 echo "Uploading report..."
 {
-	echo "meta/report.bz2"
 	echo "meta/report.txt"
 	echo "meta/report.html"
 	echo "meta/report.dot"
 	echo "meta/report.svg"
+	echo "meta/report.xz"
 	echo "meta/status"
 	while read pkg; do
 		echo "${pkg}/pre-clean.log"
diff --git a/pkgtools/pbulk/files/pbulk/scripts/upload b/pkgtools/pbulk/files/pbulk/scripts/upload
index 5131a9d386aa..9a433214392b 100755
--- a/pkgtools/pbulk/files/pbulk/scripts/upload
+++ b/pkgtools/pbulk/files/pbulk/scripts/upload
@@ -48,6 +48,7 @@ cd ${packages}
 	    echo "+ SHA512.bz2"
 	echo "+ All/pkg_summary.bz2"
 	echo "+ All/pkg_summary.gz"
+	echo "+ All/pkg_summary.xz"
 	${packages_script} ${loc} ${pkg_sufx}
 	echo "- *"
 } | LC_ALL=C sort | ${rsync} --exclude-from=- ${pkg_rsync_args} . ${pkg_rsync_target}

From 5047ac94a4453dbc99d284abe62fa3d97ab22285 Mon Sep 17 00:00:00 2001
From: Jonathan Perkin <jperkin@joyent.com>
Date: Thu, 12 Nov 2015 15:28:35 +0000
Subject: [PATCH 17/28] Add support for $script_phase_pre_upload.

This is set to a script that is executed by the upload script prior to
starting the actual upload, and is useful for performing any sanity
checks on what is about to be uploaded.

Add variables used by Joyent to ensure a certain number of packages
built ($pkg_rsync_min) and that a list of core packages all built
successfully ($pkg_rsync_required_pkgs) to the example config.
---
 pkgtools/pbulk/files/pbulk/pbulk.conf     | 11 +++++++++++
 pkgtools/pbulk/files/pbulk/scripts/upload | 11 +++++++++++
 2 files changed, 22 insertions(+)

diff --git a/pkgtools/pbulk/files/pbulk/pbulk.conf b/pkgtools/pbulk/files/pbulk/pbulk.conf
index 5a3059057cf1..ff26e0915990 100644
--- a/pkgtools/pbulk/files/pbulk/pbulk.conf
+++ b/pkgtools/pbulk/files/pbulk/pbulk.conf
@@ -53,6 +53,15 @@ report_rsync_target="pkgsrc@192.168.75.1:/public/reports/current/DragonFly-1.8"
 report_subject_prefix="pkgsrc"
 report_recipients="pkgsrc-bulk@netbsd.org"
 
+# Example variables used by an optional pre-upload script to ensure certain
+# criteria are met before publishing.
+#
+# Set a minimum number of successful packages.
+#pkg_rsync_min=10000
+#
+# Ensure all packages listed in the file built successfully.
+#pkg_rsync_required_pkgs=/required_pkgs
+
 # An archive containing the pkgsrc binary tree after bootstrapping.
 #
 bootstrapkit=/usr/pkgsrc/bootstrap/bootstrap.tar.gz
@@ -152,6 +161,8 @@ script_phase_pre_build=@PREFIX@/libexec/pbulk/pre-build
 script_phase_build=@PREFIX@/libexec/pbulk/build
 script_phase_report=@PREFIX@/libexec/pbulk/report
 script_phase_scan=@PREFIX@/libexec/pbulk/scan
+# Optional pre-upload script called from upload.
+#script_phase_pre_upload=/pre-upload
 script_phase_upload=@PREFIX@/libexec/pbulk/upload
 
 # Arguments are buildlog and pkgname
diff --git a/pkgtools/pbulk/files/pbulk/scripts/upload b/pkgtools/pbulk/files/pbulk/scripts/upload
index 9a433214392b..ad030a7d3181 100755
--- a/pkgtools/pbulk/files/pbulk/scripts/upload
+++ b/pkgtools/pbulk/files/pbulk/scripts/upload
@@ -40,6 +40,17 @@ if [ "${config_version}" != "@PBULK_CONFIG_VERSION@" ]; then
 	exit 1
 fi
 
+#
+# Run optional check script.  This allows ensuring certain criteria are met
+# before continuing with the upload process, for example to avoid publishing
+# a new package set with critical packages missing.  PBULK_CONF is passed in
+# the environment so that the script can source it for useful variables.
+#
+if [ -x "${script_phase_pre_upload}" ]; then
+	echo "Running pre-upload check script..."
+	PBULK_CONF=${PBULK_CONF:-@PBULK_CONFIG@} ${script_phase_pre_upload}
+fi
+
 echo "Uploading packages..."
 cd ${packages}
 {

From 88c003f8ff753b3c971bdfcac6c3cf073fe2ce7d Mon Sep 17 00:00:00 2001
From: Jonathan Perkin <jperkin@joyent.com>
Date: Tue, 9 Feb 2016 10:06:46 +0000
Subject: [PATCH 18/28] Add support for README.push

---
 pkgtools/pbulk/files/pbulk/scripts/create-report-html.awk | 4 ++++
 pkgtools/pbulk/files/pbulk/scripts/create-report-txt.awk  | 6 ++++++
 pkgtools/pbulk/files/pbulk/scripts/pkg-build              | 4 ++++
 3 files changed, 14 insertions(+)

diff --git a/pkgtools/pbulk/files/pbulk/scripts/create-report-html.awk b/pkgtools/pbulk/files/pbulk/scripts/create-report-html.awk
index 62fe863b3d03..0a76f4cc5ad4 100755
--- a/pkgtools/pbulk/files/pbulk/scripts/create-report-html.awk
+++ b/pkgtools/pbulk/files/pbulk/scripts/create-report-html.awk
@@ -229,6 +229,8 @@ BEGIN {
 			pkgsrc_build_start_iso = substr($0, 17)
 		else if ($0 ~ "^BUILD_END_ISO=")
 			pkgsrc_build_end_iso = substr($0, 15)
+		else if ($0 ~ "^DESCRIPTION=")
+			pkgsrc_description = substr($0, 13)
 	}
 	close(status_file)
 
@@ -271,6 +273,8 @@ BEGIN {
 	print "  </head>" > html_report
 	print "  <body>" > html_report
 	printf("    <h1> pkgsrc bulk build for %s</h1>\n", pkgsrc_platform) > html_report
+	if (pkgsrc_description)
+		printf("    <h2> Description: %s</h2>\n", pkgsrc_description) > html_report
 	printf("    <h2> Build start: %s</h2>\n", pkgsrc_build_start_iso) > html_report
 	printf("    <h2> Build end: %s</h2>\n", pkgsrc_build_end_iso) > html_report
 	print "    <hr />" > html_report
diff --git a/pkgtools/pbulk/files/pbulk/scripts/create-report-txt.awk b/pkgtools/pbulk/files/pbulk/scripts/create-report-txt.awk
index 5e2e48775aab..b32db85d9410 100755
--- a/pkgtools/pbulk/files/pbulk/scripts/create-report-txt.awk
+++ b/pkgtools/pbulk/files/pbulk/scripts/create-report-txt.awk
@@ -72,6 +72,8 @@ BEGIN {
 			pkgsrc_build_end_iso = substr($0, 15)
 		else if ($0 ~ "^BASE_URL=")
 			pkgsrc_base_url = substr($0, 10)
+		else if ($0 ~ "^DESCRIPTION=")
+			pkgsrc_description = substr($0, 13)
 	}
 	close(status_file)
 
@@ -121,6 +123,10 @@ BEGIN {
 	print "pkgsrc bulk build report" > txt_report
 	print "========================" > txt_report
 	print "" > txt_report
+	if (pkgsrc_description) {
+		print "Description: " pkgsrc_description > txt_report
+		print "" > txt_report
+	}
 	print pkgsrc_platform > txt_report
 	print "Compiler: " pkgsrc_compiler > txt_report
 	print "" > txt_report
diff --git a/pkgtools/pbulk/files/pbulk/scripts/pkg-build b/pkgtools/pbulk/files/pbulk/scripts/pkg-build
index efa9ad4d1e36..ca51c3eeca67 100755
--- a/pkgtools/pbulk/files/pbulk/scripts/pkg-build
+++ b/pkgtools/pbulk/files/pbulk/scripts/pkg-build
@@ -190,6 +190,10 @@ if [ ${ret} -eq 0 ]; then
 
 	echo "COMPILER=${compiler}" >> ${loc}/status
 	echo "PLATFORM=${opsys} ${opver}/${platform}" >> ${loc}/status
+
+	if [ -s ${pkgsrc}/README.push ]; then
+		echo "DESCRIPTION=`head -1 ${pkgsrc}/README.push`" >> ${loc}/status
+	fi
 fi
 
 ${pkg_up_to_date_script} ${pkgname} ${dependencies} && exit 0

From cf2449c1117610a9fddd873d8f6127be4e20ec30 Mon Sep 17 00:00:00 2001
From: Jonathan Perkin <jperkin@joyent.com>
Date: Mon, 21 Mar 2016 14:25:33 +0000
Subject: [PATCH 19/28] Add support for --link-dest atomic updates.

---
 pkgtools/pbulk/files/pbulk/scripts/upload | 39 ++++++++++++++++++++++-
 1 file changed, 38 insertions(+), 1 deletion(-)

diff --git a/pkgtools/pbulk/files/pbulk/scripts/upload b/pkgtools/pbulk/files/pbulk/scripts/upload
index ad030a7d3181..3cbd0484bae9 100755
--- a/pkgtools/pbulk/files/pbulk/scripts/upload
+++ b/pkgtools/pbulk/files/pbulk/scripts/upload
@@ -62,4 +62,41 @@ cd ${packages}
 	echo "+ All/pkg_summary.xz"
 	${packages_script} ${loc} ${pkg_sufx}
 	echo "- *"
-} | LC_ALL=C sort | ${rsync} --exclude-from=- ${pkg_rsync_args} . ${pkg_rsync_target}
+} | LC_ALL=C sort >/tmp/rsync-exclude-$$
+
+# Joyent specific.  Additional pbulk.conf variables provide host + directory so
+# that we can insert the rsync --link-dest argument and ensure atomic directory
+# updates.  Requires pkg_rsync_target_linkdest and pkg_rsync_target_tmpdest to
+# reside on the same file system for hardlinks.
+if [ -z "${pkg_rsync_target_host}" -o \
+     -z "${pkg_rsync_target_linkdest}" -o \
+     -z "${pkg_rsync_target_tmpdest}" ]; then
+	# Legacy sync, just performed in-place directly to target.
+	${rsync} --exclude-from=/tmp/rsync-exclude-$$ ${pkg_rsync_args} \
+	    . ${pkg_rsync_target}
+else
+	# Use link-dest to create/update to a shadow directory.  The tmpdest
+	# argument requires the usual parent directories to exist first.  If
+	# link-dest doesn't exist (e.g. an initial sync) it is not an error.
+	${rsync} --exclude-from=/tmp/rsync-exclude-$$  ${pkg_rsync_args} \
+	    --link-dest=${pkg_rsync_target_linkdest} \
+	    . ${pkg_rsync_target_host}:${pkg_rsync_target_tmpdest}
+
+	# Move any existing linkdest out of the way, promote the new tmpdest,
+	# and remove the previous linkdest.  We move it to a directory based
+	# on tmpdest as that is usually a hidden directory, avoiding directory
+	# listings showing the old directory while it is being deleted.
+	ssh ${pkg_rsync_target_host} "
+		if [ -f ${pkg_rsync_target_tmpdest}/All/pkg_summary.gz ]; then
+			if [ -d ${pkg_rsync_target_linkdest} ]; then
+				mv ${pkg_rsync_target_linkdest} ${pkg_rsync_target_tmpdest}-old
+			else
+				mkdir -p $(dirname ${pkg_rsync_target_linkdest})
+			fi
+			mv ${pkg_rsync_target_tmpdest} ${pkg_rsync_target_linkdest}
+			rm -rf ${pkg_rsync_target_tmpdest}-old
+		fi
+	"
+fi
+
+rm -f /tmp/rsync-exclude-$$

From 66a42ab2695cbac00bf2f2df2a05062935b5e2eb Mon Sep 17 00:00:00 2001
From: Jonathan Perkin <jperkin@joyent.com>
Date: Thu, 13 Oct 2016 09:56:05 +0100
Subject: [PATCH 20/28] Support $report_platform.

---
 pkgtools/pbulk/files/pbulk/scripts/pkg-build | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/pkgtools/pbulk/files/pbulk/scripts/pkg-build b/pkgtools/pbulk/files/pbulk/scripts/pkg-build
index ca51c3eeca67..1c5a2cffdc7e 100755
--- a/pkgtools/pbulk/files/pbulk/scripts/pkg-build
+++ b/pkgtools/pbulk/files/pbulk/scripts/pkg-build
@@ -189,7 +189,11 @@ if [ ${ret} -eq 0 ]; then
 	compiler=`cd ${pkgsrc}/pkgtools/pkg_install && ${make} show-var VARNAME=PKGSRC_COMPILER`
 
 	echo "COMPILER=${compiler}" >> ${loc}/status
-	echo "PLATFORM=${opsys} ${opver}/${platform}" >> ${loc}/status
+	if [ -n "${report_platform}" ]; then
+		echo "PLATFORM=${report_platform}" >> ${loc}/status
+	else
+		echo "PLATFORM=${opsys} ${opver}/${platform}" >> ${loc}/status
+	fi
 
 	if [ -s ${pkgsrc}/README.push ]; then
 		echo "DESCRIPTION=`head -1 ${pkgsrc}/README.push`" >> ${loc}/status

From d85f1a72ff248780974e1d01134f9f3b81bca252 Mon Sep 17 00:00:00 2001
From: Jonathan Perkin <jperkin@joyent.com>
Date: Thu, 13 Oct 2016 09:56:30 +0100
Subject: [PATCH 21/28] Generate pkg_files archives.

---
 pkgtools/pbulk/files/pbulk/scripts/build     | 6 ++++++
 pkgtools/pbulk/files/pbulk/scripts/pkg-build | 2 ++
 pkgtools/pbulk/files/pbulk/scripts/upload    | 3 +++
 3 files changed, 11 insertions(+)

diff --git a/pkgtools/pbulk/files/pbulk/scripts/build b/pkgtools/pbulk/files/pbulk/scripts/build
index b3ebf0000853..cb8a57908415 100755
--- a/pkgtools/pbulk/files/pbulk/scripts/build
+++ b/pkgtools/pbulk/files/pbulk/scripts/build
@@ -61,6 +61,9 @@ BUILD_END_EPOCHE=%s' >> ${loc}/status
 
 echo "Building pkg_summary..."
 cd ${packages}/All
+sed -e 's,^,../pkginfo/,' -e 's/$/.pkglist/' < ${loc}/success | sort | xargs cat 2>/dev/null | ${gzip} -c > pkg_files.gz
+${gzip} -dc < pkg_files.gz | ${bzip2} -c > pkg_files.bz2
+${gzip} -dc < pkg_files.gz | ${xz} -zc > pkg_files.xz
 sed -e 's,^,../pkginfo/,' -e 's/$/.pkginfo/' < ${loc}/success | sort | xargs cat 2>/dev/null | ${gzip} -c > pkg_summary.gz
 ${gzip} -dc < pkg_summary.gz | ${bzip2} -c > pkg_summary.bz2
 ${gzip} -dc < pkg_summary.gz | ${xz} -zc > pkg_summary.xz
@@ -70,6 +73,9 @@ if [ "${checksum_packages}" != "no" ] && \
 	echo "Building SHA512..."
 	cd ${packages}
 	{
+		${digest} SHA512 All/pkg_files.bz2
+		${digest} SHA512 All/pkg_files.gz
+		${digest} SHA512 All/pkg_files.xz
 		${digest} SHA512 All/pkg_summary.bz2
 		${digest} SHA512 All/pkg_summary.gz
 		${digest} SHA512 All/pkg_summary.xz
diff --git a/pkgtools/pbulk/files/pbulk/scripts/pkg-build b/pkgtools/pbulk/files/pbulk/scripts/pkg-build
index 1c5a2cffdc7e..30da8c9eb2dd 100755
--- a/pkgtools/pbulk/files/pbulk/scripts/pkg-build
+++ b/pkgtools/pbulk/files/pbulk/scripts/pkg-build
@@ -318,6 +318,8 @@ if [ -z "${restricted}" ]; then
 		    > ${packages}/checksums/${pkgname}.sha512 || cleanup
 		${pkg_info} -X All/${pkgname}${pkg_sufx} \
 		    > ${packages}/pkginfo/${pkgname}.pkginfo || cleanup
+		${pkg_info} -L All/${pkgname}${pkg_sufx} \
+		    > ${packages}/pkginfo/${pkgname}.pkglist || cleanup
 	)
 fi
 
diff --git a/pkgtools/pbulk/files/pbulk/scripts/upload b/pkgtools/pbulk/files/pbulk/scripts/upload
index 3cbd0484bae9..425c44894a2e 100755
--- a/pkgtools/pbulk/files/pbulk/scripts/upload
+++ b/pkgtools/pbulk/files/pbulk/scripts/upload
@@ -57,6 +57,9 @@ cd ${packages}
 	[ "${checksum_packages}" != "no" ] && \
 	    [ "${checksum_packages}" != "NO" ] && \
 	    echo "+ SHA512.bz2"
+	echo "+ All/pkg_files.bz2"
+	echo "+ All/pkg_files.gz"
+	echo "+ All/pkg_files.xz"
 	echo "+ All/pkg_summary.bz2"
 	echo "+ All/pkg_summary.gz"
 	echo "+ All/pkg_summary.xz"

From 57057f95475e40fdb422f793dacf6aa2c3fa333c Mon Sep 17 00:00:00 2001
From: Jonathan Perkin <jperkin@joyent.com>
Date: Wed, 18 Jan 2017 11:38:04 +0000
Subject: [PATCH 22/28] Category links are no more, remove support for them.

---
 .../files/pbulk/scripts/compute-packages.awk      | 15 +--------------
 pkgtools/pbulk/files/pbulk/scripts/upload         |  1 +
 2 files changed, 2 insertions(+), 14 deletions(-)

diff --git a/pkgtools/pbulk/files/pbulk/scripts/compute-packages.awk b/pkgtools/pbulk/files/pbulk/scripts/compute-packages.awk
index deb5dba744ce..4c4240ed6db5 100755
--- a/pkgtools/pbulk/files/pbulk/scripts/compute-packages.awk
+++ b/pkgtools/pbulk/files/pbulk/scripts/compute-packages.awk
@@ -53,9 +53,6 @@ BEGIN {
 			pkgs[cur] = cur
 		}
 
-		if ($0 ~ "^CATEGORIES=")
-			categories[cur] = substr($0, 12)
-
 		if ($0 ~ "^BUILD_STATUS=")
 			status[cur] = substr($0, 14)
 
@@ -82,17 +79,7 @@ BEGIN {
 		# skip restricted packages
 		if (pkg in restricted)
 			continue;
-		# build category/file list
-		split(categories[pkg], cats, "[ \t]+")
-		cats[0] = "All"
-		for (cat_idx in cats) {
-			cat = cats[cat_idx]
-			if (!(cat in printed_cats)) {
-				print "+ " cat "/"
-				printed_cats[cat] = cat
-			}
-			print "+ " cat "/" pkg pkg_sufx
-		}
+		print "+ All/" pkg pkg_sufx
 	}
 	close(success_file)
 }
diff --git a/pkgtools/pbulk/files/pbulk/scripts/upload b/pkgtools/pbulk/files/pbulk/scripts/upload
index 425c44894a2e..8614063a872a 100755
--- a/pkgtools/pbulk/files/pbulk/scripts/upload
+++ b/pkgtools/pbulk/files/pbulk/scripts/upload
@@ -57,6 +57,7 @@ cd ${packages}
 	[ "${checksum_packages}" != "no" ] && \
 	    [ "${checksum_packages}" != "NO" ] && \
 	    echo "+ SHA512.bz2"
+	echo "+ All/"
 	echo "+ All/pkg_files.bz2"
 	echo "+ All/pkg_files.gz"
 	echo "+ All/pkg_files.xz"

From e4c373bb47db51b35c1a00545303d959cbd9c29c Mon Sep 17 00:00:00 2001
From: Jonathan Perkin <jperkin@joyent.com>
Date: Thu, 19 Jan 2017 16:29:56 +0000
Subject: [PATCH 23/28] Ensure the options cache directory is created.

---
 pkgtools/pbulk/files/pbulk/scripts/pre-build | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pkgtools/pbulk/files/pbulk/scripts/pre-build b/pkgtools/pbulk/files/pbulk/scripts/pre-build
index 52f37809a125..6bdbcea7e268 100755
--- a/pkgtools/pbulk/files/pbulk/scripts/pre-build
+++ b/pkgtools/pbulk/files/pbulk/scripts/pre-build
@@ -82,7 +82,7 @@ else
 	rm -rf "${bulklog}"/* || true
 fi
 
-mkdir -p "${bulklog}" "${loc}"
+mkdir -p "${bulklog}" "${bulklog}/cache" "${loc}"
 
 case "${master_mode}" in
 [nN][oO])

From af11172775d4ed76d6c6cf78e0d0bfc77f18f79f Mon Sep 17 00:00:00 2001
From: Jonathan Perkin <jperkin@joyent.com>
Date: Thu, 19 Jan 2017 16:33:09 +0000
Subject: [PATCH 24/28] Use --partial-dir to avoid rsync mirror failures.

---
 pkgtools/pbulk/files/pbulk/scripts/upload | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/pkgtools/pbulk/files/pbulk/scripts/upload b/pkgtools/pbulk/files/pbulk/scripts/upload
index 8614063a872a..1949e0f9a3c2 100755
--- a/pkgtools/pbulk/files/pbulk/scripts/upload
+++ b/pkgtools/pbulk/files/pbulk/scripts/upload
@@ -77,13 +77,13 @@ if [ -z "${pkg_rsync_target_host}" -o \
      -z "${pkg_rsync_target_tmpdest}" ]; then
 	# Legacy sync, just performed in-place directly to target.
 	${rsync} --exclude-from=/tmp/rsync-exclude-$$ ${pkg_rsync_args} \
-	    . ${pkg_rsync_target}
+	    --partial-dir=.rsync-partial . ${pkg_rsync_target}
 else
 	# Use link-dest to create/update to a shadow directory.  The tmpdest
 	# argument requires the usual parent directories to exist first.  If
 	# link-dest doesn't exist (e.g. an initial sync) it is not an error.
-	${rsync} --exclude-from=/tmp/rsync-exclude-$$  ${pkg_rsync_args} \
-	    --link-dest=${pkg_rsync_target_linkdest} \
+	${rsync} --exclude-from=/tmp/rsync-exclude-$$ ${pkg_rsync_args} \
+	    --partial-dir=.rsync-partial --link-dest=${pkg_rsync_target_linkdest} \
 	    . ${pkg_rsync_target_host}:${pkg_rsync_target_tmpdest}
 
 	# Move any existing linkdest out of the way, promote the new tmpdest,

From 44cd608239777069124eddb4ae51db4139e2ae66 Mon Sep 17 00:00:00 2001
From: Jonathan Perkin <jperkin@joyent.com>
Date: Thu, 25 May 2017 12:03:35 +0100
Subject: [PATCH 25/28] Add parallelism to pkg_{files,summary} creation.

---
 pkgtools/pbulk/files/pbulk/scripts/build | 25 ++++++++++++++++++------
 1 file changed, 19 insertions(+), 6 deletions(-)

diff --git a/pkgtools/pbulk/files/pbulk/scripts/build b/pkgtools/pbulk/files/pbulk/scripts/build
index cb8a57908415..e886518cea5c 100755
--- a/pkgtools/pbulk/files/pbulk/scripts/build
+++ b/pkgtools/pbulk/files/pbulk/scripts/build
@@ -61,12 +61,25 @@ BUILD_END_EPOCHE=%s' >> ${loc}/status
 
 echo "Building pkg_summary..."
 cd ${packages}/All
-sed -e 's,^,../pkginfo/,' -e 's/$/.pkglist/' < ${loc}/success | sort | xargs cat 2>/dev/null | ${gzip} -c > pkg_files.gz
-${gzip} -dc < pkg_files.gz | ${bzip2} -c > pkg_files.bz2
-${gzip} -dc < pkg_files.gz | ${xz} -zc > pkg_files.xz
-sed -e 's,^,../pkginfo/,' -e 's/$/.pkginfo/' < ${loc}/success | sort | xargs cat 2>/dev/null | ${gzip} -c > pkg_summary.gz
-${gzip} -dc < pkg_summary.gz | ${bzip2} -c > pkg_summary.bz2
-${gzip} -dc < pkg_summary.gz | ${xz} -zc > pkg_summary.xz
+{
+	sed -e 's,^,../pkginfo/,' -e 's/$/.pkglist/' < ${loc}/success \
+	    | sort | xargs cat 2>/dev/null > pkg_files
+	cat pkg_files | ${gzip} -c > pkg_files.gz &
+	cat pkg_files | ${bzip2} -c > pkg_files.bz2 &
+	cat pkg_files | ${xz} -zc > pkg_files.xz &
+	wait
+	rm -f pkg_files
+} &
+{
+	sed -e 's,^,../pkginfo/,' -e 's/$/.pkginfo/' < ${loc}/success \
+	    | sort | xargs cat 2>/dev/null > pkg_summary
+	cat pkg_summary | ${gzip} -c > pkg_summary.gz &
+	cat pkg_summary | ${bzip2} -c > pkg_summary.bz2 &
+	cat pkg_summary | ${xz} -zc > pkg_summary.xz &
+	wait
+	rm -f pkg_summary
+} &
+wait
 
 if [ "${checksum_packages}" != "no" ] && \
    [ "${checksum_packages}" != "NO" ]; then

From e631a19a9a4294bf0600119dfb81ab3dcfb6d4f9 Mon Sep 17 00:00:00 2001
From: Jonathan Perkin <jperkin@joyent.com>
Date: Mon, 3 Jul 2017 14:09:35 +0100
Subject: [PATCH 26/28] Send report mail after finishing report upload.

---
 pkgtools/pbulk/files/pbulk/scripts/report | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/pkgtools/pbulk/files/pbulk/scripts/report b/pkgtools/pbulk/files/pbulk/scripts/report
index 498c3135a920..ce5c1ff7836d 100755
--- a/pkgtools/pbulk/files/pbulk/scripts/report
+++ b/pkgtools/pbulk/files/pbulk/scripts/report
@@ -73,9 +73,6 @@ if [ -z "${build_start_iso}" ] || [ -z "${build_start_dir}" ]; then
 	exit 1
 fi
 
-echo "Sending report mail..."
-cat ${loc}/report.txt | ${mail} -s "${report_subject_prefix} ${platform} ${build_start_iso}" ${report_recipients}
-
 cd ${bulklog}
 echo "Uploading report..."
 {
@@ -123,3 +120,6 @@ echo "Uploading report..."
 		${rsync} --exclude-from=- ${report_rsync_args} . ${report_rsync_target}/${build_start_dir}
 	fi
 }
+
+echo "Sending report mail..."
+cat ${loc}/report.txt | ${mail} -s "${report_subject_prefix} ${platform} ${build_start_iso}" ${report_recipients}

From 0e6c1daa6411c3d9c16c8e4c56c5ef489c6a3bf1 Mon Sep 17 00:00:00 2001
From: Jonathan Perkin <jperkin@joyent.com>
Date: Wed, 5 Jul 2017 16:12:14 +0100
Subject: [PATCH 27/28] fix "set -e" failure in new pkg_summary generation.

---
 pkgtools/pbulk/files/pbulk/scripts/build | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/pkgtools/pbulk/files/pbulk/scripts/build b/pkgtools/pbulk/files/pbulk/scripts/build
index e886518cea5c..49831dcb63ba 100755
--- a/pkgtools/pbulk/files/pbulk/scripts/build
+++ b/pkgtools/pbulk/files/pbulk/scripts/build
@@ -63,7 +63,7 @@ echo "Building pkg_summary..."
 cd ${packages}/All
 {
 	sed -e 's,^,../pkginfo/,' -e 's/$/.pkglist/' < ${loc}/success \
-	    | sort | xargs cat 2>/dev/null > pkg_files
+	    | sort | xargs cat 2>/dev/null > pkg_files || true
 	cat pkg_files | ${gzip} -c > pkg_files.gz &
 	cat pkg_files | ${bzip2} -c > pkg_files.bz2 &
 	cat pkg_files | ${xz} -zc > pkg_files.xz &
@@ -72,7 +72,7 @@ cd ${packages}/All
 } &
 {
 	sed -e 's,^,../pkginfo/,' -e 's/$/.pkginfo/' < ${loc}/success \
-	    | sort | xargs cat 2>/dev/null > pkg_summary
+	    | sort | xargs cat 2>/dev/null > pkg_summary || true
 	cat pkg_summary | ${gzip} -c > pkg_summary.gz &
 	cat pkg_summary | ${bzip2} -c > pkg_summary.bz2 &
 	cat pkg_summary | ${xz} -zc > pkg_summary.xz &

From 4571b5cdf4cccd46bad52b188468a622d19737da Mon Sep 17 00:00:00 2001
From: Jonathan Perkin <jperkin@joyent.com>
Date: Thu, 7 Nov 2019 10:24:53 +0000
Subject: [PATCH 28/28] Support manta-sync if MANTA_SYNC_ARGS is set.

This will be much faster for rebuilds, and avoids some issues with
muntar relating to subusers, but does require manually excluding all
files that are normally handled by this script until manta-sync adds
support for --exclude-from.

We also apply some manual retries in lieu of client support.
---
 pkgtools/pbulk/files/pbulk/scripts/report | 51 +++++++++++++++++++----
 1 file changed, 44 insertions(+), 7 deletions(-)

diff --git a/pkgtools/pbulk/files/pbulk/scripts/report b/pkgtools/pbulk/files/pbulk/scripts/report
index ce5c1ff7836d..3192a88a1d6a 100755
--- a/pkgtools/pbulk/files/pbulk/scripts/report
+++ b/pkgtools/pbulk/files/pbulk/scripts/report
@@ -109,13 +109,50 @@ echo "Uploading report..."
 		# Variables sourced from pbulk.conf
 		export MANTA_USER MANTA_KEY_ID MANTA_URL MANTA_SUBUSER
 		PATH=${MANTA_PATH}:${PATH}
-		# Output directory only needs to not exist or be empty.
-		tmpdir="${bulklog}/pbulk-rsync-dir"
-		tmptar="${bulklog}/pbulk-rsync.tar"
-		rm -rf ${tmpdir} ${tmptar}
-		${rsync} -rn --exclude-from=- --out-format='%n' . ${tmpdir} \
-		    | grep -v '/$' | @TAR@ -cf ${tmptar} -T -
-		muntar -f ${tmptar} ${report_manta_target}/${build_start_dir}
+		#
+		# Use manta-sync if the args have been set.  manta-sync does
+		# not (yet) support --exclude-from, so a manual list of -x
+		# exclusions needs to be supplied instead of using stdin.
+		#
+		# Using 'noglob' makes things a lot simpler with expansion and
+		# globs in variables.
+		#
+		# Moving the 'report' file is a hack due to manta-sync not
+		# currently having a way to limit excludes, so "-x report" will
+		# exclude report.* which is definitely not what we want!
+		#
+		if [ -n "${MANTA_SYNC_ARGS}" ]; then
+			set -o noglob
+			mv ${loc}/report ${loc}/report.save
+			#
+			# Run it a few times to catch any temporarily failures,
+			# the client does not yet support retries.
+			#
+			sync_failed=true
+			for retry in 1 2 3 4 5; do
+				echo "manta-sync attempt ${retry}..."
+				manta-sync ${MANTA_SYNC_ARGS} . ${report_manta_target}/${build_start_dir} || continue
+				sync_failed=false
+				break
+			done
+			set +o noglob
+			if ${sync_failed}; then
+				echo "All sync attempts failed, aborting..."
+				exit 1
+			fi
+		#
+		# Otherwise fall back to muntar, using rsync to create a
+		# temporary tar to unpack without excluded files.
+		#
+		else
+			# Output directory only needs to not exist or be empty.
+			tmpdir="${bulklog}/pbulk-rsync-dir"
+			tmptar="${bulklog}/pbulk-rsync.tar"
+			rm -rf ${tmpdir} ${tmptar}
+			${rsync} -rn --exclude-from=- --out-format='%n' . ${tmpdir} \
+			    | grep -v '/$' | @TAR@ -cf ${tmptar} -T -
+			muntar -f ${tmptar} ${report_manta_target}/${build_start_dir}
+		fi
 	else
 		${rsync} --exclude-from=- ${report_rsync_args} . ${report_rsync_target}/${build_start_dir}
 	fi