Permalink
Browse files

[RFC Experimental] Use redo instead of make for package dependencies.

This has several advantages:

- We don't have to include all package .mk files in every run:
        - Which greatly reduces make startup time
        - And greatly reduces global variable pollution

- Fully interoperates with make (including sharing -j jobserver tokens)
  so we can get most of the benefits of redo in just a small patch,
  without rewriting everything.

- Does not affect any of the syntax for package or rootfs definitions.

- We can express more complicated dependencies, like "if packages A and
  B are enabled, always build B after A, but it's okay if only one is
  enabled."

- We can eventually add more complicated dependency resolution rules,
  like fetching a pre-built binary package from a repository if the
  right conditions are satisfied, and falling back to source build
  otherwise.

- With the new redo-log command, redo has more readable, repeatable,
  linearized logs for parallel build logs than any other build system.
  • Loading branch information...
apenwarr committed Oct 16, 2018
1 parent 45498bb commit 229d3b1bdf23a340187b461fbad901beb25bdc4d
@@ -1,10 +1,13 @@
/show-targets
/all-install-stamps
/output
/dl
/.auto.deps
/.config.cmd
/.config.old
/..config.tmp
/.config
/.redo
*.depend
*.o
/*.patch
@@ -13,3 +16,5 @@
*.rej
*~
*.pyc
*.deps
*.redo*.tmp
@@ -83,8 +83,9 @@ _all:
else # umask / $(CURDIR) / $(O)
# This is our default rule, so must come first
all:
.PHONY: all
all:
+redo all
# Set and export the version string
export BR2_VERSION := 2018.11-git
@@ -129,7 +130,7 @@ export BR2_VERSION_FULL := $(BR2_VERSION)$(shell $(TOPDIR)/support/scripts/setlo
# List of targets and target patterns for which .config doesn't need to be read in
noconfig_targets := menuconfig nconfig gconfig xconfig config oldconfig randconfig \
defconfig %_defconfig allyesconfig allnoconfig alldefconfig syncconfig release \
randpackageconfig allyespackageconfig allnopackageconfig \
randpackageconfig allyespackageconfig allnopackageconfig all \
print-version olddefconfig distclean manual manual-% check-package
# Some global targets do not trigger a build, but are used to collect
@@ -141,7 +142,7 @@ noconfig_targets := menuconfig nconfig gconfig xconfig config oldconfig randconf
# We're building in two situations: when MAKECMDGOALS is empty
# (default target is to build), or when MAKECMDGOALS contains
# something else than one of the nobuild_targets.
nobuild_targets := source %-source \
nobuild_targets := all source %-source \
legal-info %-legal-info external-deps _external-deps \
clean distclean help show-targets graph-depends \
%-graph-depends %-show-depends %-show-version \
@@ -486,8 +487,6 @@ export BASE_DIR
#
################################################################################
all: world
# Include legacy before the other things, because package .mk files
# may rely on it.
include Makefile.legacy
@@ -517,7 +516,23 @@ ifneq ($(PACKAGE_OVERRIDE_FILE),)
-include $(PACKAGE_OVERRIDE_FILE)
endif
ifneq ($(ONE_PACKAGE_FILE),)
# FIXME: these packages define variables silently depended upon by other packages.
# Eventually, it would be better to explicitly include .mk files depended upon by
# any given package from that package itself, so we can avoid hardcoding this list.
include $(filter-out $(ONE_PACKAGE_FILE), \
package/lzip/lzip.mk \
package/autoconf/autoconf.mk \
package/automake/automake.mk \
package/gettext/gettext.mk \
package/pkgconf/pkgconf.mk \
package/mtd/mtd.mk \
$(sort $(wildcard package/linux-headers/*.mk package/gcc/*.mk)) \
)
include $(filter-out toolchain/% package/gcc/%, $(ONE_PACKAGE_FILE))
else
include $(sort $(wildcard package/*/*.mk))
endif
include boot/common.mk
include linux/linux.mk
@@ -694,13 +709,12 @@ endef
TARGET_FINALIZE_HOOKS += PURGE_LOCALES
endif
$(TARGETS_ROOTFS): target-finalize
# Avoid the rootfs name leaking down the dependency chain
target-finalize: ROOTFS=
.PHONY: target-finalize
target-finalize: $(PACKAGES)
target-finalize:
+redo-ifchange $(PACKAGES)
@$(call MESSAGE,"Finalizing target directory")
# Check files that are touched by more than one package
./support/scripts/check-uniq-files -t target $(BUILD_DIR)/packages-file-list.txt
@@ -868,7 +882,7 @@ else # ifeq ($(BR2_HAVE_DOT_CONFIG),y)
# rule for it.
# Also for 'all' we error out and ask the user to configure first.
.PHONY: linux toolchain
linux toolchain all: outputmakefile
linux toolchain: outputmakefile
$(error Please configure Buildroot first (e.g. "make menuconfig"))
@exit 1
@@ -0,0 +1,22 @@
# Find the .do file in do.rules to build any given target.
( cd do.rules && redo-whichdo "$1" ) | {
ifcreate=
while read dofile; do
dopath="./do.rules/$dofile"
if [ -e "$dopath" ]; then
redo-ifcreate $ifcreate
redo-ifchange "$dopath"
suffix=
[ "$dofile" != "${dofile#default.}" ] && suffix=${dofile#default}
suffix=${suffix%.do}
x2=${1%"$suffix"}
set -- "$1" "$x2" "$3"
. "$dopath"
exit
else
ifcreate="$ifcreate $dopath"
fi
done
echo "Fatal: no .do file found for $1" >&2
exit 44
}
@@ -0,0 +1,18 @@
BR2_x86_64=y
BR2_x86_corei7=y
BR2_TOOLCHAIN_BUILDROOT_LOCALE=y
BR2_TOOLCHAIN_BUILDROOT_CXX=y
BR2_PACKAGE_ZLIB=y
BR2_PACKAGE_NETTLE=y
BR2_PACKAGE_OPENSSL=y
BR2_PACKAGE_LIBRESSL=y
BR2_PACKAGE_LIBTASN1=y
BR2_PACKAGE_PCRE=y
BR2_TARGET_ROOTFS_CLOOP=y
BR2_TARGET_ROOTFS_CPIO=y
BR2_TARGET_ROOTFS_CRAMFS=y
BR2_TARGET_ROOTFS_JFFS2=y
BR2_TARGET_ROOTFS_ROMFS=y
BR2_TARGET_ROOTFS_SQUASHFS=y
BR2_TARGET_ROOTFS_UBI=y
BR2_TARGET_ROOTFS_YAFFS2=y
@@ -0,0 +1,23 @@
BR2_x86_64=y
BR2_x86_corei7=y
BR2_TOOLCHAIN_EXTERNAL=y
BR2_TOOLCHAIN_EXTERNAL_PATH="$(PWD)/../buildroot/tc/host"
BR2_TOOLCHAIN_EXTERNAL_GCC_7=y
BR2_TOOLCHAIN_EXTERNAL_HEADERS_4_18=y
BR2_TOOLCHAIN_EXTERNAL_LOCALE=y
# BR2_TOOLCHAIN_EXTERNAL_HAS_THREADS_DEBUG is not set
BR2_TOOLCHAIN_EXTERNAL_CXX=y
BR2_PACKAGE_ZLIB=y
BR2_PACKAGE_NETTLE=y
BR2_PACKAGE_OPENSSL=y
BR2_PACKAGE_LIBRESSL=y
BR2_PACKAGE_LIBTASN1=y
BR2_PACKAGE_PCRE=y
BR2_TARGET_ROOTFS_CLOOP=y
BR2_TARGET_ROOTFS_CPIO=y
BR2_TARGET_ROOTFS_CRAMFS=y
BR2_TARGET_ROOTFS_JFFS2=y
BR2_TARGET_ROOTFS_ROMFS=y
BR2_TARGET_ROOTFS_SQUASHFS=y
BR2_TARGET_ROOTFS_UBI=y
BR2_TARGET_ROOTFS_YAFFS2=y
@@ -0,0 +1,6 @@
redo-always
for d in output/build/*/.stamp_target_installed; do
[ -e "$d" ] && echo "$d"
done >$3
redo-stamp <$3
@@ -0,0 +1,16 @@
. ./helpers.od
redo-ifchange show-targets syncconfig
{
packages= roots=
while read target; do
if startswith "$target" "rootfs-"; then
roots="$roots $target"
else
packages="$packages $target"
fi
done
} <show-targets
redo-ifchange $packages
redo-ifchange $roots
@@ -0,0 +1,5 @@
. ./package.od
redo-ifchange "$2.configure"
make "$pkg-build" ONE_PACKAGE_FILE="$makefile" >&2
redo-ifchange output/build/$pkg*/.stamp_built
@@ -0,0 +1,5 @@
. ./package.od
redo-ifchange "$2.depends"
make "$pkg-configure" ONE_PACKAGE_FILE="$makefile" >&2
redo-ifchange output/build/$pkg*/.stamp_configured
@@ -0,0 +1,5 @@
. ./package.od
redo-ifchange "$2.patch" # FIXME: do we really need to patch first?
redo-ifchange "$2.deps"
exec xargs redo-ifchange <$2.deps
@@ -0,0 +1,6 @@
. ./package.od
redo-ifchange "$makefile"
make --no-print-directory "$pkg-show-depends" ONE_PACKAGE_FILE=$makefile |
sed -e 's/\s/\n/g' >$3
cat "$3" >&2
@@ -0,0 +1,4 @@
. ./package.od
redo-always
exec make "$pkg-dirclean" ONE_PACKAGE_FILE="$makefile" >&2
@@ -0,0 +1,46 @@
exec >&2
. ./helpers.od
redo-ifchange helpers.od
if contains "$1" "/"; then
die "No .do file for '$1'"
fi
if startswith "$1" "rootfs-"; then
pkg=${1#rootfs-}
makefile=fs/$pkg/$pkg.mk
[ -e "$makefile" ] || die "no buildroot package found: $makefile"
exec redo-ifchange fs/$pkg.rootfs
fi
pbase=$(basename "$1")
p_x=$(echo "$pbase" | sed -e 's/-\(source\|extract\|patch\|depends\|configure\|build\|install\|show-depends\|show-rdepends\|show-recursive-depends\|show-recursive-rdepends\|graph-depends\|graph-rdepends\|dirclean\|reconfigure\|rebuild\)$/ \1/')
pkg=${p_x%% *}
suffix=${p_x#"$pkg"}
suffix=${suffix# }
# FIXME: reject host-* packages that don't define any host- build rules
host=
startswith "$pkg" "host-" && host=.host
pkg=${pkg#host-}
#echo "package='$pkg' suffix='$suffix'" >&2
makefile=$(find package toolchain/toolchain*/ -name "$pkg.mk" -print)
[ -n "$makefile" ] || die "no buildroot package found: .../$pkg/$pkg.mk"
pkgdir=${makefile%/*}
#echo "makefile='$makefile' pkgdir='$pkgdir'" >&2
redo-ifchange syncconfig
[ "$suffix" != "" ] || suffix=install
if [ "$suffix" = install ]; then
# These aren't necessary, but help reduce the level of recursion.
# (Otherwise $pkg.install depends on .build, which depends on .configure,
# etc, and the call tree is needlessly deep, which uglifies the logs.)
redo-ifchange "$pkgdir$host.source"
redo-ifchange "$pkgdir$host.extract"
redo-ifchange "$pkgdir$host.patch"
redo-ifchange "$pkgdir$host.depends"
redo-ifchange "$pkgdir$host.configure"
redo-ifchange "$pkgdir$host.build"
fi
exec redo-ifchange "$pkgdir$host.$suffix"
@@ -0,0 +1,9 @@
. ./package.od
# FIXME: auto-depend only on the necessary extractor packages
[ "$pkg" != 'host-lzip' -a "$pkg" != 'host-skeleton' ] &&
redo-ifchange host-lzip
redo-ifchange "$2.source"
make "$pkg-extract" ONE_PACKAGE_FILE="$makefile" >&2
redo-ifchange output/build/$pkg*/.stamp_extracted
@@ -0,0 +1,4 @@
. ./package.od
redo-always
exec make "$pkg-graph-depends" >&2
@@ -0,0 +1,4 @@
. ./package.od
redo-always
exec make "$pkg-graph-rdepends" >&2
@@ -0,0 +1,9 @@
. ./package.od
redo-ifchange "$2.build"
make "$pkg" ONE_PACKAGE_FILE="$makefile" >&2
if [ -n "$host" ]; then
redo-ifchange output/build/$pkg*/.stamp_host_installed
else
redo-ifchange output/build/$pkg*/.stamp_target_installed
fi
@@ -0,0 +1,10 @@
. ./package.od
redo-ifchange "$2.extract"
for d in $2/*.patch $2/*.diff; do
[ -e "$d" ] || continue
echo "patchfile: '$d'" >&2
echo "$d"
done | xargs redo-ifchange
make "$pkg-patch" ONE_PACKAGE_FILE="$makefile" >&2
redo-ifchange output/build/$pkg*/.stamp_patched
@@ -0,0 +1,5 @@
. ./package.od
redo-always
redo-ifchange "$2.configure"
exec make "$pkg-rebuild" >&2
@@ -0,0 +1,5 @@
. ./package.od
redo-always
redo-ifchange "$2.depends"
exec make "$pkg-reconfigure" >&2
@@ -0,0 +1,6 @@
. ./package.od
redo-ifchange rootfs-common
redo-ifchange all-install-stamps
xargs redo-ifchange <all-install-stamps
make "rootfs-$pkg" ONE_PACKAGE_FILE="$makefile" >&2
@@ -0,0 +1,4 @@
. ./package.od
redo-always
exec make "$pkg-show-depends" ONE_PACKAGE_FILE="$makefile" >&2
@@ -0,0 +1,4 @@
. ./package.od
redo-always
exec make "$pkg-show-rdepends" >&2
@@ -0,0 +1,4 @@
. ./package.od
redo-always
exec make "$pkg-show-recursive-depends" >&2
@@ -0,0 +1,4 @@
. ./package.od
redo-always
exec make "$pkg-show-recursive-rdepends" >&2
@@ -0,0 +1,4 @@
. ./package.od
make "$pkg-source" ONE_PACKAGE_FILE="$makefile" >&2
redo-ifchange output/build/$pkg*/.stamp_downloaded
@@ -0,0 +1,5 @@
. ./helpers.od
make _rootfs-common >&2
redo-ifchange all-install-stamps host-fakeroot
xargs redo-ifchange <all-install-stamps
@@ -0,0 +1,5 @@
redo-ifchange .config
make --no-print-directory show-targets |
sed -e 's/\s/\n/g' >$3
redo-stamp <$3
cat $3 >&2
@@ -0,0 +1,4 @@
# FIXME: there should be a better way to rely on "basic setup" being done.
make syncconfig >&2
make dirs output/build/buildroot-config/auto.conf >&2
redo-ifchange output/build/buildroot-config/auto.conf
@@ -0,0 +1,2 @@
redo python-cssselect bash procps-ng busybox \
thttpd xz linux-pam oracle-mysql sqlite
Oops, something went wrong.

0 comments on commit 229d3b1

Please sign in to comment.