Skip to content

Commit

Permalink
Merge branch 'joyent/feature/pbulk/trunk' into joyent/release/trunk
Browse files Browse the repository at this point in the history
  • Loading branch information
Jonathan Perkin committed Jan 14, 2021
2 parents 0c062b5 + 4571b5c commit b8a3978
Show file tree
Hide file tree
Showing 27 changed files with 550 additions and 107 deletions.
2 changes: 1 addition & 1 deletion mk/pbulk/pbulk-index.mk
Original file line number Diff line number Diff line change
Expand Up @@ -117,7 +117,7 @@ pbulk-index-item:
@echo "USE_DESTDIR="${_USE_DESTDIR:Q}
@echo "BOOTSTRAP_PKG="${BOOTSTRAP_PKG}
@echo "USERGROUP_PHASE="${USERGROUP_PHASE:Q}
@echo "SCAN_DEPENDS="${_PBULK_SCAN_DEPENDS:O:u:Q}
@echo "SCAN_DEPENDS="${_PBULK_SCAN_DEPENDS:N${PKGBUILD_BASEDIR:U/nonexistent}/*:O:u:Q}
.if defined(_PBULK_MULTI_NEEDED)
@printf "MULTI_VERSION="
.for _t in ${_PBULK_MULTI_NEEDED}
Expand Down
6 changes: 3 additions & 3 deletions pkgtools/pbulk/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ COMMENT= Modular bulk build framework
.include "../../pkgtools/pbulk/Makefile.common"

USE_TOOLS+= awk:run bzip2:run digest:run gzip:run make:run \
mail:run sed:run tar:run
mail:run sed:run tar:run xz:run
DEPENDS+= rsync-[0-9]*:../../net/rsync
DEPENDS+= pbulk-base>=0.38:../../pkgtools/pbulk-base

Expand All @@ -25,7 +25,7 @@ SUBST_STAGE.tools= pre-configure
SUBST_MESSAGE.tools= Fixing references to tools
SUBST_FILES.tools= pbulk.conf scripts/build scripts/build-client-start \
scripts/bulkbuild scripts/bulkbuild-rebuild scripts/bulkbuild-restart \
scripts/client-clean scripts/client-prepare \
scripts/chroot scripts/client-clean scripts/client-prepare \
scripts/pkg-build scripts/pkg-up-to-date scripts/pre-build \
scripts/report scripts/scan scripts/scan-client-start scripts/upload \
scripts/compute-packages.awk scripts/create-broken-graph.awk \
Expand All @@ -34,7 +34,7 @@ SUBST_FILES.tools= pbulk.conf scripts/build scripts/build-client-start \
SUBST_VARS.tools= AWK BZIP2 CHOWN DIGEST GZIP_CMD ID MAIL_CMD NEATO \
PBULK_CONFIG PBULK_CONFIG_VERSION PKG_ADD_CMD \
PKG_ADMIN_CMD PKG_DELETE_CMD PKG_INFO_CMD PREFIX \
SED SH TAR TARGET_MAKE
SED SH TAR TARGET_MAKE XZ

CONF_FILES+= share/examples/pbulk/pbulk.conf ${PKG_SYSCONFDIR}/pbulk.conf

Expand Down
1 change: 1 addition & 0 deletions pkgtools/pbulk/PLIST
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ bin/bulkbuild-rebuild
bin/bulkbuild-restart
libexec/pbulk/build
libexec/pbulk/build-client-start
libexec/pbulk/chroot
libexec/pbulk/client-clean
libexec/pbulk/client-prepare
libexec/pbulk/compute-packages
Expand Down
34 changes: 34 additions & 0 deletions pkgtools/pbulk/files/pbulk/pbulk.conf
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,23 @@ build_clients="192.168.75.21 192.168.75.22 192.168.75.23 192.168.75.24"
master_port_scan=${master_ip}:2001
master_port_build=${master_ip}:2002

# Create chroots for each phase. This allows concurrent build/scan on the
# same host. chroot_create/chroot_delete are scripts which take a single
# argument which is the chroot directory to create/delete, and chroot_dir is
# used as the prefix for the name of the chroot directory.
#
# To use chroots on a single host, set master_mode=yes and leave both
# build_clients and scan_clients unset or empty.
#
#chroot_create=/path/to/mksandbox
#chroot_delete=/path/to/rmsandbox
#chroot_dir=/chroot/pkgsrc

# To enable concurrent chroots, set build_chroots/scan_chroots > 1.
#
#build_chroots=4
#scan_chroots=8

# Some flags for the programs that publish the binary packages and the
# build report. If you want to disable the publishing at all, see below.
#
Expand All @@ -36,6 +53,15 @@ report_rsync_target="pkgsrc@192.168.75.1:/public/reports/current/DragonFly-1.8"
report_subject_prefix="pkgsrc"
report_recipients="pkgsrc-bulk@netbsd.org"

# Example variables used by an optional pre-upload script to ensure certain
# criteria are met before publishing.
#
# Set a minimum number of successful packages.
#pkg_rsync_min=10000
#
# Ensure all packages listed in the file built successfully.
#pkg_rsync_required_pkgs=/required_pkgs

# An archive containing the pkgsrc binary tree after bootstrapping.
#
bootstrapkit=/usr/pkgsrc/bootstrap/bootstrap.tar.gz
Expand Down Expand Up @@ -102,6 +128,7 @@ neato=@NEATO@
rsync=@PREFIX@/bin/rsync
sed=@SED@
tar=@TAR@
xz=@XZ@

loc=${bulklog}/meta

Expand All @@ -119,6 +146,7 @@ keep_prefix=no
client_prepare=@PREFIX@/libexec/pbulk/client-prepare
client_prepare_action=@PREFIX@/libexec/pbulk/client-clean
client_prepare_chroot=:
chroot_script=@PREFIX@/libexec/pbulk/chroot
pkg_up_to_date_script=@PREFIX@/libexec/pbulk/pkg-up-to-date
pbuild_script=@PREFIX@/libexec/pbulk/pkg-build
pbuild_start_script=@PREFIX@/libexec/pbulk/build-client-start
Expand All @@ -133,6 +161,8 @@ script_phase_pre_build=@PREFIX@/libexec/pbulk/pre-build
script_phase_build=@PREFIX@/libexec/pbulk/build
script_phase_report=@PREFIX@/libexec/pbulk/report
script_phase_scan=@PREFIX@/libexec/pbulk/scan
# Optional pre-upload script called from upload.
#script_phase_pre_upload=/pre-upload
script_phase_upload=@PREFIX@/libexec/pbulk/upload

# Arguments are buildlog and pkgname
Expand All @@ -142,3 +172,7 @@ sync_buildlog=:
sync_package=default_sync_package
# PKG_SUFX as set in mk.conf
pkg_sufx=.tgz

# Optional local override scripts.
#
#client_clean_local=/client_clean_local
24 changes: 24 additions & 0 deletions pkgtools/pbulk/files/pbulk/pscan/client.c
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,30 @@ find_full_tree_client(int fd, const char *bmake_path)
if (cat_len == 0)
break;

/*
* Check for SUBDIR including a '/', if so then it has been
* passed via USER_ADDITIONAL_PKGS from the top-level Makefile,
* and should be added as a package path directly.
*/
if (memchr(cat, '/', cat_len) != NULL) {
if (len_pkgs == allocated_pkgs) {
if (allocated_pkgs == 0) {
allocated_pkgs = 1024;
pkgs = xmalloc(sizeof(*pkgs) *
allocated_pkgs);
} else {
allocated_pkgs *= 2;
pkgs = xrealloc(pkgs,
sizeof(*pkgs) * allocated_pkgs);
}
}
pkgs[len_pkgs] = xasprintf("%.*s", (int)cat_len, cat);
len_pkgs_data += strlen(pkgs[len_pkgs]) + 1;
++len_pkgs;
cat += cat_len;
continue;
}

cat_path = xasprintf("%s/%.*s", pkgsrc_tree, (int)cat_len, cat);
buf_orig = buf = read_from_child(cat_path, bmake_path, extract_subdir);
free(cat_path);
Expand Down
10 changes: 10 additions & 0 deletions pkgtools/pbulk/files/pbulk/pscan/jobs.c
Original file line number Diff line number Diff line change
Expand Up @@ -278,6 +278,16 @@ add_job(const char *cat, size_t cat_len, const char *dir, size_t dir_len)
free(location);
}

void
add_job_pkgpath(const char *pkgpath, size_t pkgpath_len)
{
char *location;

location = xasprintf("%.*s", (int)pkgpath_len, pkgpath);
add_job_full(location);
free(location);
}

void
add_job_full(const char *location)
{
Expand Down
11 changes: 11 additions & 0 deletions pkgtools/pbulk/files/pbulk/pscan/pscan.c
Original file line number Diff line number Diff line change
Expand Up @@ -225,6 +225,17 @@ find_full_tree(void)
if (cat_len == 0)
break;

/*
* Check for SUBDIR including a '/', if so then it has been
* passed via USER_ADDITIONAL_PKGS from the top-level Makefile,
* and should be added as a package path directly.
*/
if (memchr(cat, '/', cat_len) != NULL) {
add_job_pkgpath(cat, cat_len);
cat += cat_len;
continue;
}

cat_path = xasprintf("%s/%.*s", pkgsrc_tree, (int)cat_len, cat);
buf_orig = buf = read_from_child(cat_path, bmake_path, extract_subdir);
free(cat_path);
Expand Down
1 change: 1 addition & 0 deletions pkgtools/pbulk/files/pbulk/pscan/pscan.h
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@ void client_mode(const char *, const char *);
void master_mode(const char *, const char *);

void add_job(const char *, size_t, const char *, size_t);
void add_job_pkgpath(const char *, size_t);
void add_job_full(const char *);
struct scan_job *get_job(void);
void process_job(struct scan_job *, enum job_state);
Expand Down
2 changes: 1 addition & 1 deletion pkgtools/pbulk/files/pbulk/scripts/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

SCRIPTS= build build-client-start bulkbuild bulkbuild-rebuild \
bulkbuild-restart \
client-clean client-prepare \
chroot client-clean client-prepare \
pkg-build pkg-up-to-date pre-build report \
scan scan-client-start upload \
compute-packages.awk create-broken-graph.awk \
Expand Down
37 changes: 30 additions & 7 deletions pkgtools/pbulk/files/pbulk/scripts/build
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@
# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
# SUCH DAMAGE.

. @PBULK_CONFIG@
. ${PBULK_CONF:-@PBULK_CONFIG@}

set -e

Expand All @@ -46,7 +46,9 @@ case "${master_mode}" in
${pbuild} -r ${loc}/pbuild -v -b ${pbuild_script} ${loc}/presolve ${loc}/success ${loc}/error
;;
[yY][eE][sS])
[ -n "${chroot_dir}" ] && ${chroot_script} build create ${chroot_dir}
${pbuild} -r ${loc}/pbuild -I ${pbuild_start_script} -m ${master_port_build} -v ${loc}/presolve ${loc}/success ${loc}/error
[ -n "${chroot_dir}" ] && ${chroot_script} build delete ${chroot_dir}
;;
*)
echo "master_mode must be either yes or no."
Expand All @@ -59,16 +61,37 @@ BUILD_END_EPOCHE=%s' >> ${loc}/status

echo "Building pkg_summary..."
cd ${packages}/All
sed 's/$/.tgz/' < ${loc}/success | sort | xargs ${pkg_info} -X | ${gzip} -c > pkg_summary.gz
${gzip} -dc < pkg_summary.gz | ${bzip2} -c > pkg_summary.bz2
{
sed -e 's,^,../pkginfo/,' -e 's/$/.pkglist/' < ${loc}/success \
| sort | xargs cat 2>/dev/null > pkg_files || true
cat pkg_files | ${gzip} -c > pkg_files.gz &
cat pkg_files | ${bzip2} -c > pkg_files.bz2 &
cat pkg_files | ${xz} -zc > pkg_files.xz &
wait
rm -f pkg_files
} &
{
sed -e 's,^,../pkginfo/,' -e 's/$/.pkginfo/' < ${loc}/success \
| sort | xargs cat 2>/dev/null > pkg_summary || true
cat pkg_summary | ${gzip} -c > pkg_summary.gz &
cat pkg_summary | ${bzip2} -c > pkg_summary.bz2 &
cat pkg_summary | ${xz} -zc > pkg_summary.xz &
wait
rm -f pkg_summary
} &
wait

if [ "${checksum_packages}" != "no" ] && \
[ "${checksum_packages}" != "NO" ]; then
echo "Building SHA512..."
cd ${packages}
{
echo "All/pkg_summary.bz2"
echo "All/pkg_summary.gz"
sed 's|^\(.*\)$|All/\1.tgz|' < ${loc}/success
} | sort | xargs ${digest} SHA512 | ${bzip2} -c > SHA512.bz2
${digest} SHA512 All/pkg_files.bz2
${digest} SHA512 All/pkg_files.gz
${digest} SHA512 All/pkg_files.xz
${digest} SHA512 All/pkg_summary.bz2
${digest} SHA512 All/pkg_summary.gz
${digest} SHA512 All/pkg_summary.xz
sed -e 's,^,checksums/,' -e 's/$/.sha512/' < ${loc}/success | xargs cat 2>/dev/null
} | sort | ${bzip2} -c > SHA512.bz2
fi
10 changes: 8 additions & 2 deletions pkgtools/pbulk/files/pbulk/scripts/build-client-start
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
#!@SH@
# $NetBSD: build-client-start,v 1.5 2016/12/18 22:59:35 joerg Exp $

. @PBULK_CONFIG@
. ${PBULK_CONF:-@PBULK_CONFIG@}

set -e

Expand Down Expand Up @@ -29,7 +29,13 @@ for client in ${build_clients}; do
;;
esac
if [ -z "$path" ]; then
ssh $port $client "${pbuild} -c ${master_port_build} -b ${pbuild_script}" &
if [ -n "${chroot_create}" -a "${chroot_dir}" ]; then
ssh $port $client "for dir in ${chroot_dir}-build*; do
PBULK_CONF=${PBULK_CONF} chroot \${dir} @SH@ -c \"${pbuild} -c ${master_port_build} -b ${pbuild_script}\" &
done" &
else
ssh $port $client "${pbuild} -c ${master_port_build} -b ${pbuild_script}" &
fi
else
chroot "$path" ${pbuild} -c ${master_port_build} -b ${pbuild_script} &
fi
Expand Down
16 changes: 15 additions & 1 deletion pkgtools/pbulk/files/pbulk/scripts/bulkbuild
Original file line number Diff line number Diff line change
@@ -1,7 +1,21 @@
#!@SH@
# $NetBSD: bulkbuild,v 1.1.1.1 2007/06/19 19:49:59 joerg Exp $

. @PBULK_CONFIG@
if [ "$1" = "-h" ]; then
echo "usage: $0 [-h] [<config file>]"
exit 2
elif [ -f "$1" ]; then
case "$1" in
/*)
export PBULK_CONF="$1"; shift
;;
*)
export PBULK_CONF="$(pwd)/$1"; shift
;;
esac
fi

. ${PBULK_CONF:-@PBULK_CONFIG@}

set -e

Expand Down
18 changes: 16 additions & 2 deletions pkgtools/pbulk/files/pbulk/scripts/bulkbuild-rebuild
Original file line number Diff line number Diff line change
@@ -1,7 +1,21 @@
#!@SH@
# $NetBSD: bulkbuild-rebuild,v 1.3 2008/09/16 18:21:30 joerg Exp $

. @PBULK_CONFIG@
if [ "$1" = "-h" ]; then
echo "Usage: $0 [-h] [<config file>] package ..."
exit 2
elif [ -f "$1" ]; then
case "$1" in
/*)
export PBULK_CONF="$1"; shift
;;
*)
export PBULK_CONF="$(pwd)/$1"; shift
;;
esac
fi

. ${PBULK_CONF:-@PBULK_CONFIG@}

set -e

Expand Down Expand Up @@ -33,7 +47,7 @@ build_pkg() {
}

if [ $# = 0 ]; then
echo "Usage: $0 package ..."
echo "Usage: $0 [-h] [<config file>] package ..."
fi

while [ $# != 0 ]; do
Expand Down
16 changes: 15 additions & 1 deletion pkgtools/pbulk/files/pbulk/scripts/bulkbuild-restart
Original file line number Diff line number Diff line change
@@ -1,7 +1,21 @@
#!@SH@
# $NetBSD: bulkbuild-restart,v 1.4 2016/12/18 22:38:16 joerg Exp $

. @PBULK_CONFIG@
if [ "$1" = "-h" ]; then
echo "usage: $0 [-h] [<config file>]"
exit 2
elif [ -f "$1" ]; then
case "$1" in
/*)
export PBULK_CONF="$1"; shift
;;
*)
export PBULK_CONF="$(pwd)/$1"; shift
;;
esac
fi

. ${PBULK_CONF:-@PBULK_CONFIG@}

set -e

Expand Down
Loading

0 comments on commit b8a3978

Please sign in to comment.