diff --git a/lib/chroot-buildpackages.sh b/lib/chroot-buildpackages.sh index 4a498bb43910..68f59da293b7 100644 --- a/lib/chroot-buildpackages.sh +++ b/lib/chroot-buildpackages.sh @@ -46,18 +46,33 @@ create_chroot() components['impish']='main,universe,multiverse' components['jammy']='main,universe,multiverse' display_alert "Creating build chroot" "$release/$arch" "info" - local includes="ccache,locales,git,ca-certificates,devscripts,libfile-fcntllock-perl,debhelper,rsync,python3,distcc" + local includes="ccache,locales,git,ca-certificates,devscripts,libfile-fcntllock-perl,debhelper,rsync,python3,distcc,apt-utils" + # perhaps a temporally workaround - [[ $release == buster || $release == bullseye || $release == focal || $release == hirsute || $release == sid ]] && includes=${includes}",perl-openssl-defaults,libnet-ssleay-perl" + case $release in + buster|bullseye|focal|hirsute|sid) + includes=${includes}",perl-openssl-defaults,libnet-ssleay-perl" + ;; + esac + if [[ $NO_APT_CACHER != yes ]]; then local mirror_addr="http://localhost:3142/${apt_mirror[${release}]}" else local mirror_addr="http://${apt_mirror[${release}]}" fi + mkdir -p "${target_dir}" cd "${target_dir}" - debootstrap --variant=buildd --components="${components[${release}]}" --arch="${arch}" $DEBOOTSTRAP_OPTION --foreign --include="${includes}" "${release}" "${target_dir}" "${mirror_addr}" - [[ $? -ne 0 || ! -f "${target_dir}"/debootstrap/debootstrap ]] && exit_with_error "Create chroot first stage failed" + + debootstrap --variant=buildd \ + --components="${components[${release}]}" \ + --arch="${arch}" $DEBOOTSTRAP_OPTION \ + --foreign \ + --include="${includes}" "${release}" "${target_dir}" "${mirror_addr}" + + [[ $? -ne 0 || ! -f "${target_dir}"/debootstrap/debootstrap ]] && \ + exit_with_error "Create chroot first stage failed" + cp /usr/bin/${qemu_binary[$arch]} "${target_dir}"/usr/bin/ [[ ! -f "${target_dir}"/usr/share/keyrings/debian-archive-keyring.gpg ]] && \ mkdir -p "${target_dir}"/usr/share/keyrings/ && \ @@ -65,6 +80,7 @@ create_chroot() chroot "${target_dir}" /bin/bash -c "/debootstrap/debootstrap --second-stage" [[ $? -ne 0 || ! -f "${target_dir}"/bin/bash ]] && exit_with_error "Create chroot second stage failed" + create_sources_list "$release" "${target_dir}" [[ $NO_APT_CACHER != yes ]] && \ echo 'Acquire::http { Proxy "http://localhost:3142"; };' > "${target_dir}"/etc/apt/apt.conf.d/02proxy @@ -72,8 +88,10 @@ create_chroot() APT::Install-Recommends "0"; APT::Install-Suggests "0"; EOF - [[ -f "${target_dir}"/etc/locale.gen ]] && sed -i "s/^# en_US.UTF-8/en_US.UTF-8/" "${target_dir}"/etc/locale.gen + [[ -f "${target_dir}"/etc/locale.gen ]] && \ + sed -i "s/^# en_US.UTF-8/en_US.UTF-8/" "${target_dir}"/etc/locale.gen chroot "${target_dir}" /bin/bash -c "locale-gen; update-locale LANG=en_US:en LC_ALL=en_US.UTF-8" + printf '#!/bin/sh\nexit 101' > "${target_dir}"/usr/sbin/policy-rc.d chmod 755 "${target_dir}"/usr/sbin/policy-rc.d rm "${target_dir}"/etc/resolv.conf 2>/dev/null @@ -86,7 +104,17 @@ create_chroot() mkdir -p "${target_dir}"/var/lock fi chroot "${target_dir}" /bin/bash -c "/usr/sbin/update-ccache-symlinks" - [[ $release == bullseye || $release == focal || $release == hirsute || $release == sid ]] && chroot "${target_dir}" /bin/bash -c "ln -s /usr/bin/python3 /usr/bin/python" + + display_alert "Upgrading packages in" "${target_dir}" "info" + chroot "${target_dir}" /bin/bash -c "apt-get -q update; apt-get -q -y upgrade; apt-get clean" + date +%s >"$target_dir/root/.update-timestamp" + + case $release in + bullseye|focal|hirsute|sid) + chroot "${target_dir}" /bin/bash -c "apt-get install python-is-python3" + ;; + esac + touch "${target_dir}"/root/.debootstrap-complete display_alert "Debootstrap complete" "${release}/${arch}" "info" } ############################################################################# @@ -147,27 +175,62 @@ chroot_build_packages() for arch in $target_arch; do display_alert "Starting package building process" "$release/$arch" "info" - local target_dir - target_dir="${SRC}/cache/buildpkg/${release}-${arch}-v${CHROOT_CACHE_VERSION}" + local t_name=${release}-${arch}-v${CHROOT_CACHE_VERSION} local distcc_bindaddr="127.0.0.2" - [[ ! -f "${target_dir}"/root/.debootstrap-complete ]] && create_chroot "${target_dir}" "${release}" "${arch}" - [[ ! -f "${target_dir}"/root/.debootstrap-complete ]] && exit_with_error "Creating chroot failed" "${release}/${arch}" + # Create a clean environment archive if it does not exist. + if [ ! -f "${SRC}/cache/buildpkg/${t_name}.tar.xz" ]; then + local tmp_dir=$(mktemp -d "${SRC}"/.tmp/debootstrap-XXXXX) + create_chroot "${tmp_dir}/${t_name}" "${release}" "${arch}" + display_alert "Create a clean Environment archive" "${t_name}.tar.xz" "info" + ( + tar -cp --directory="${tmp_dir}/" ${t_name} \ + | pv -p -b -r -s "$(du -sb "${tmp_dir}/${t_name}" | cut -f1)" \ + | pixz -4 >"${SRC}/cache/buildpkg/${t_name}.tar.xz" + ) + rm -rf $tmp_dir + fi + + # Unpack the clean environment archive, if it exists. + if [ -f "${SRC}/cache/buildpkg/${t_name}.tar.xz" ]; then + local tmp_dir=$(mktemp -d "${SRC}"/.tmp/build-XXXXX) + ( cd $tmp_dir + display_alert "Unpack the clean environment" "${t_name}.tar.xz" "info" + tar -xJf "${SRC}/cache/buildpkg/${t_name}.tar.xz" || \ + exit_with_error "Is not extracted" "${SRC}/cache/buildpkg/${t_name}.tar.xz" + ) + target_dir="$tmp_dir/${t_name}" + else + exit_with_error "Creating chroot failed" "${release}/${arch}" + fi - [[ -f /var/run/distcc/"${release}-${arch}".pid ]] && kill "$(<"/var/run/distcc/${release}-${arch}.pid")" > /dev/null 2>&1 + [[ -f /var/run/distcc/"${release}-${arch}".pid ]] && + kill "$(<"/var/run/distcc/${release}-${arch}.pid")" > /dev/null 2>&1 chroot_prepare_distccd "${release}" "${arch}" # DISTCC_TCP_DEFER_ACCEPT=0 - DISTCC_CMDLIST=/tmp/distcc/${release}-${arch}/cmdlist TMPDIR=/tmp/distcc distccd --daemon \ - --pid-file "/var/run/distcc/${release}-${arch}.pid" --listen $distcc_bindaddr --allow 127.0.0.0/24 \ + DISTCC_CMDLIST=/tmp/distcc/${release}-${arch}/cmdlist \ + TMPDIR=/tmp/distcc distccd --daemon \ + --pid-file "/var/run/distcc/${release}-${arch}.pid" \ + --listen $distcc_bindaddr --allow 127.0.0.0/24 \ --log-file "/tmp/distcc/${release}-${arch}.log" --user distccd + [[ -d $target_dir ]] || + exit_with_error "Clean Environment is not visible" "$target_dir" + local t=$target_dir/root/.update-timestamp if [[ ! -f ${t} || $(( ($(date +%s) - $(<"${t}")) / 86400 )) -gt 7 ]]; then display_alert "Upgrading packages" "$release/$arch" "info" systemd-nspawn -a -q -D "${target_dir}" /bin/bash -c "apt-get -q update; apt-get -q -y upgrade; apt-get clean" date +%s > "${t}" + display_alert "Repack a clean Environment archive after upgrading" "${t_name}.tar.xz" "info" + rm "${SRC}/cache/buildpkg/${t_name}.tar.xz" + ( + tar -cp --directory="${tmp_dir}/" ${t_name} \ + | pv -p -b -r -s "$(du -sb "${tmp_dir}/${t_name}" | cut -f1)" \ + | pixz -4 >"${SRC}/cache/buildpkg/${t_name}.tar.xz" + ) fi for plugin in "${SRC}"/packages/extras-buildpkgs/*.conf; do @@ -200,12 +263,29 @@ chroot_build_packages() display_alert "Packages are up to date" "$package_name $release/$arch" "info" continue fi + + # Delete the environment if there was a build in it. + # And unpack the clean environment again. + if [[ -f "${target_dir}"/root/build.sh ]] && [[ -d $tmp_dir ]]; then + rm -rf $tmp_dir + local tmp_dir=$(mktemp -d "${SRC}"/.tmp/build-XXXXX) + ( cd $tmp_dir + display_alert "Unpack the clean environment" "${t_name}.tar.xz" "info" + tar -xJf "${SRC}/cache/buildpkg/${t_name}.tar.xz" || \ + exit_with_error "Is not extracted" "${SRC}/cache/buildpkg/${t_name}.tar.xz" + ) + target_dir="$tmp_dir/${t_name}" + fi + display_alert "Building packages" "$package_name $release/$arch" "ext" + ts=$(date +%s) local dist_builddeps_name="package_builddeps_${release}" [[ -v $dist_builddeps_name ]] && package_builddeps="${package_builddeps} ${!dist_builddeps_name}" # create build script + LOG_OUTPUT_FILE=/root/build-"${package_name}".log create_build_script + unset LOG_OUTPUT_FILE fetch_from_repo "$package_repo" "extra/$package_name" "$package_ref" @@ -223,7 +303,12 @@ chroot_build_packages() built_ok+=("$package_name:$release/$arch") fi mv "${target_dir}"/root/*.deb "${plugin_target_dir}" 2>/dev/null + mv "${target_dir}"/root/*.log "$DEST/${LOG_SUBPATH}/" + te=$(date +%s) + display_alert "Build time $package_name " " $(($te - $ts)) sec." "info" done + # Delete a temporary directory + if [ -d $tmp_dir ]; then rm -rf $tmp_dir;fi # cleanup for distcc kill $(/dev/null | \ - grep '^ii' | \ - awk -F '|' '{print \$2}' | \ - cut -d ':' -f 1 - ) - - for packet in $package_builddeps - do - grep -q -x -e "\$packet" <<< "\$installed" || deps+=("\$packet") - done - - if [[ \${#deps[@]} -gt 0 ]]; then - display_alert "Installing build dependencies" - apt-get -y -q update - apt-get -y -q \ - --no-install-recommends \ - --show-progress \ - -o DPKG::Progress-Fancy=1 install "\${deps[@]}" - fi - fi + LOG_OUTPUT_FILE=$LOG_OUTPUT_FILE + $(declare -f install_pkg_deb) + cd /root/build display_alert "Copying sources" rsync -aq /root/sources/"${package_name}" /root/build/ @@ -296,6 +359,16 @@ create_build_script () # copy overlay / "debianization" files [[ -d "/root/overlay/${package_name}/" ]] && rsync -aq /root/overlay/"${package_name}" /root/build/ + package_builddeps="$package_builddeps" + if [ -z "\$package_builddeps" ]; then + # Calculate build dependencies by a standard function and + # еxclude special comparison characters like "|" "(>= 9)" + package_builddeps="\$(dpkg-checkbuilddeps |& awk -F":" '{gsub(/[|]|[(].*[)]/, " ", \$0); print \$NF}')" + fi + if [[ -n "\${package_builddeps}" ]]; then + install_pkg_deb \${package_builddeps} + fi + # set upstream version [[ -n "${package_upstream_version}" ]] && debchange --preserve --newversion "${package_upstream_version}" "Import from upstream" @@ -304,7 +377,9 @@ create_build_script () debchange -l~armbian"${REVISION}"+ "Custom $VENDOR release" display_alert "Building package" - dpkg-buildpackage -b -us -j2 + # Set the number of build threads and certainly send + # the standard error stream to the log file. + dpkg-buildpackage -b -us -j${NCPU_CHROOT:-2} 2>>\$LOG_OUTPUT_FILE if [[ \$? -eq 0 ]]; then cd /root/build diff --git a/lib/general.sh b/lib/general.sh index e08f28e11dba..97acc1f6b341 100644 --- a/lib/general.sh +++ b/lib/general.sh @@ -24,10 +24,13 @@ # addtorepo # repo-remove-old-packages # wait_for_package_manager +# install_pkg_deb # prepare_host_basic # prepare_host # webseed # download_and_verify +# show_developer_warning +# show_checklist_variables # cleaning @@ -1173,6 +1176,92 @@ wait_for_package_manager() +# Installing debian packages in the armbian build system. +# +# list="upgrade clean pkg1 pkg2 pkg3 pkgbadname pkg-1.0 | pkg-2.0 pkg5 (>= 9)" +# install_pkg_deb $list +# +# If the package has a bad name, we will see it in the log file. +# If there is an LOG_OUTPUT_FILE variable and it has a value as +# the full real path to the log file, then all the information will be there. +# +# The LOG_OUTPUT_FILE variable must be defined in the calling function +# before calling the install_pkg_deb function and unset after. +# +install_pkg_deb () +{ + local list="" + local log_file + local for_install + local need_upgrade=false + local need_clean=false + local _line=${BASH_LINENO[0]} + local _function=${FUNCNAME[1]} + local _file=$(basename "${BASH_SOURCE[1]}") + local tmp_file=$(mktemp /tmp/install_log_XXXXX) + export DEBIAN_FRONTEND=noninteractive + + list=$( + for p in $*;do + case $p in + upgrade) need_upgrade=true; continue ;; + clean) need_clean=true; continue ;; + \||\(*|*\)) continue ;; + esac + echo " $p" + done + ) + + if [ -d $(dirname $LOG_OUTPUT_FILE) ]; then + log_file=${LOG_OUTPUT_FILE} + else + log_file="${SRC}/output/${LOG_SUBPATH}/install.log" + fi + + echo -e "\nInstalling packages in function: $_function" "[$_file:$_line]" >>$log_file + echo -e "\nIncoming list:" >>$log_file + printf "%-30s %-30s %-30s %-30s\n" $list >>$log_file + echo "" >>$log_file + + ( apt-get -qq update + if $need_upgrade; then apt-get -y upgrade;fi + ) 2>>$log_file + + # If the package is not installed, check the latest + # up-to-date version in the apt cache. + # Exclude bad package names and send a message to the log. + for_install=$( + for p in $list;do + if $(dpkg-query -W -f '${db:Status-Abbrev}' $p |& awk '/ii/{exit 1}');then + apt-cache show $p -o APT::Cache::AllVersions=no |& \ + awk -v p=$p -v tmp_file=$tmp_file \ + '/^Package:/{print $2} /^E:/{print "Bad package name: ",p >>tmp_file}' + fi + done + ) + + # This information should be logged. + if [ -s $tmp_file ]; then + cat $tmp_file >>$log_file + fi + + if [ -n "$for_install" ]; then + + apt-get install -qq -y --no-install-recommends $for_install 2>>$log_file + + # We will show the status after installation + echo -e "\nstatus after installation:" >>$log_file + dpkg-query -W \ + -f '${binary:Package;-27} ${Version;-23} [ ${Status} ]\n' \ + $for_install >>$log_file + fi + + if $need_clean;then apt-get clean; fi + rm $tmp_file +} + + + # prepare_host_basic # # * installs only basic packages @@ -1737,7 +1826,7 @@ show_checklist_variables () for var in $checklist;do eval pval=\$$var echo -e "\n$var =:" >>$log_file - if [ $(echo "$pval" | gawk -F"/" '{print NF}') -ge 4 ];then + if [ $(echo "$pval" | awk -F"/" '{print NF}') -ge 4 ];then printf "%s\n" $pval >>$log_file else printf "%-30s %-30s %-30s %-30s\n" $pval >>$log_file diff --git a/packages/extras-buildpkgs/90-hostapd-realtek.conf b/packages/extras-buildpkgs/90-hostapd-realtek.conf index 7d86df661e2f..4ca3afbba1b2 100644 --- a/packages/extras-buildpkgs/90-hostapd-realtek.conf +++ b/packages/extras-buildpkgs/90-hostapd-realtek.conf @@ -3,10 +3,6 @@ local package_name="hostapd-realtek" local package_repo="http://w1.fi/hostap.git" local package_ref="tag:hostap_2_5" local package_upstream_version="3:2.5-4" -local package_builddeps="pkg-config libreadline-dev libpcsclite-dev libnl-route-3-dev libnl-genl-3-dev libnl-3-dev libncurses5-dev libdbus-1-dev docbook-utils docbook-to-man" -local package_builddeps_jessie="libssl-dev" -local package_builddeps_xenial="libssl-dev" -local package_builddeps_stretch="libssl1.0-dev" local package_install_target="hostapd-realtek" local package_component="${release}-utils" diff --git a/packages/extras-buildpkgs/90-hostapd.conf b/packages/extras-buildpkgs/90-hostapd.conf index 502081f80dd3..e14bdb9de539 100644 --- a/packages/extras-buildpkgs/90-hostapd.conf +++ b/packages/extras-buildpkgs/90-hostapd.conf @@ -3,7 +3,6 @@ local package_name="hostapd" local package_repo="http://w1.fi/hostap.git" local package_ref="tag:hostap_2_9" local package_upstream_version="3:2.9-102" -local package_builddeps="pkg-config libssl-dev libreadline-dev libpcsclite-dev libnl-route-3-dev libnl-genl-3-dev libnl-3-dev libncurses5-dev libdbus-1-dev docbook-utils docbook-to-man" local package_install_target="hostapd" local package_component="${release}-utils" diff --git a/packages/extras-buildpkgs/90-sunxi-tools.conf b/packages/extras-buildpkgs/90-sunxi-tools.conf index 43ea27afd1b4..b6943c739577 100644 --- a/packages/extras-buildpkgs/90-sunxi-tools.conf +++ b/packages/extras-buildpkgs/90-sunxi-tools.conf @@ -3,7 +3,7 @@ local package_name="sunxi-tools" local package_repo="https://github.com/linux-sunxi/sunxi-tools.git" local package_ref="branch:master" local package_upstream_version="1.4.2-2" -local package_builddeps="libusb-1.0-0-dev ruby binutils-arm-none-eabi pkg-config u-boot-tools zlib1g-dev libfdt-dev" + local package_install_target="sunxi-tools" local package_component="${release}-utils" diff --git a/packages/extras-buildpkgs/sunxi-tools/debian/control b/packages/extras-buildpkgs/sunxi-tools/debian/control index 65b24c48eae9..2a301deaeb16 100644 --- a/packages/extras-buildpkgs/sunxi-tools/debian/control +++ b/packages/extras-buildpkgs/sunxi-tools/debian/control @@ -2,8 +2,14 @@ Source: sunxi-tools Section: utils Priority: optional Maintainer: Ian Campbell -Build-Depends: debhelper (>= 9), pkg-config, libusb-1.0-0-dev, u-boot-tools, - ruby, binutils-arm-none-eabi, zlib1g-dev +Build-Depends: debhelper (>= 9), + pkg-config, + u-boot-tools, + binutils-arm-none-eabi, + ruby, + libusb-1.0-0-dev, + libfdt-dev, + zlib1g-dev Standards-Version: 3.9.5 Vcs-Git: git://anonscm.debian.org/collab-maint/sunxi-tools.git Vcs-Browser: http://anonscm.debian.org/cgit/collab-maint/sunxi-tools.git