Install debian packages using a special function (armbian#3368)

* Add new function install_pkg_deb Signed-off-by: The-going <48602507+The-going@users.noreply.github.com> * Use the install_pkg_deb function Signed-off-by: The-going <48602507+The-going@users.noreply.github.com> * sunxi-tools: Calculate build dependencies Signed-off-by: The-going <48602507+The-going@users.noreply.github.com> * hostapd, hostapd-realtek: Calculate build dependencies Signed-off-by: The-going <48602507+The-going@users.noreply.github.com> * Clean Environment for building as default * upgrading: Repack a clean Environment archive Signed-off-by: The-going <48602507+The-going@users.noreply.github.com>
rpardini · Dec 27, 2021 · fedc9c1 · fedc9c1
1 parent e453e4d
commit fedc9c1
Show file tree

Hide file tree

Showing 6 changed files with 213 additions and 48 deletions.
diff --git a/lib/chroot-buildpackages.sh b/lib/chroot-buildpackages.sh
@@ -46,34 +46,52 @@ create_chroot()
 	components['impish']='main,universe,multiverse'
 	components['jammy']='main,universe,multiverse'
 	display_alert "Creating build chroot" "$release/$arch" "info"
-	local includes="ccache,locales,git,ca-certificates,devscripts,libfile-fcntllock-perl,debhelper,rsync,python3,distcc"
+	local includes="ccache,locales,git,ca-certificates,devscripts,libfile-fcntllock-perl,debhelper,rsync,python3,distcc,apt-utils"
+
 	# perhaps a temporally workaround
-	[[ $release == buster || $release == bullseye || $release == focal || $release == hirsute || $release == sid ]] && includes=${includes}",perl-openssl-defaults,libnet-ssleay-perl"
+	case $release in
+		buster|bullseye|focal|hirsute|sid)
+			includes=${includes}",perl-openssl-defaults,libnet-ssleay-perl"
+		;;
+	esac
+
 	if [[ $NO_APT_CACHER != yes ]]; then
 		local mirror_addr="http://localhost:3142/${apt_mirror[${release}]}"
 	else
 		local mirror_addr="http://${apt_mirror[${release}]}"
 	fi
+
 	mkdir -p "${target_dir}"
 	cd "${target_dir}"
-	debootstrap --variant=buildd --components="${components[${release}]}" --arch="${arch}" $DEBOOTSTRAP_OPTION --foreign --include="${includes}" "${release}" "${target_dir}" "${mirror_addr}"
-	[[ $? -ne 0 || ! -f "${target_dir}"/debootstrap/debootstrap ]] && exit_with_error "Create chroot first stage failed"
+
+	debootstrap --variant=buildd \
+				--components="${components[${release}]}" \
+				--arch="${arch}" $DEBOOTSTRAP_OPTION \
+				--foreign \
+				--include="${includes}" "${release}" "${target_dir}" "${mirror_addr}"
+
+	[[ $? -ne 0 || ! -f "${target_dir}"/debootstrap/debootstrap ]] && \
+		exit_with_error "Create chroot first stage failed"
+
 	cp /usr/bin/${qemu_binary[$arch]} "${target_dir}"/usr/bin/
 	[[ ! -f "${target_dir}"/usr/share/keyrings/debian-archive-keyring.gpg ]] && \
 		mkdir -p  "${target_dir}"/usr/share/keyrings/ && \
 		cp /usr/share/keyrings/debian-archive-keyring.gpg "${target_dir}"/usr/share/keyrings/
 
 	chroot "${target_dir}" /bin/bash -c "/debootstrap/debootstrap --second-stage"
 	[[ $? -ne 0 || ! -f "${target_dir}"/bin/bash ]] && exit_with_error "Create chroot second stage failed"
+
 	create_sources_list "$release" "${target_dir}"
 	[[ $NO_APT_CACHER != yes ]] && \
 		echo 'Acquire::http { Proxy "http://localhost:3142"; };' > "${target_dir}"/etc/apt/apt.conf.d/02proxy
 	cat <<-EOF > "${target_dir}"/etc/apt/apt.conf.d/71-no-recommends
 	APT::Install-Recommends "0";
 	APT::Install-Suggests "0";
 	EOF
-	[[ -f "${target_dir}"/etc/locale.gen ]] && sed -i "s/^# en_US.UTF-8/en_US.UTF-8/" "${target_dir}"/etc/locale.gen
+	[[ -f "${target_dir}"/etc/locale.gen ]] && \
+	sed -i "s/^# en_US.UTF-8/en_US.UTF-8/" "${target_dir}"/etc/locale.gen
 	chroot "${target_dir}" /bin/bash -c "locale-gen; update-locale LANG=en_US:en LC_ALL=en_US.UTF-8"
+
 	printf '#!/bin/sh\nexit 101' > "${target_dir}"/usr/sbin/policy-rc.d
 	chmod 755 "${target_dir}"/usr/sbin/policy-rc.d
 	rm "${target_dir}"/etc/resolv.conf 2>/dev/null
@@ -86,7 +104,17 @@ create_chroot()
 		mkdir -p "${target_dir}"/var/lock
 	fi
 	chroot "${target_dir}" /bin/bash -c "/usr/sbin/update-ccache-symlinks"
-	[[ $release == bullseye || $release == focal || $release == hirsute || $release == sid ]] && chroot "${target_dir}" /bin/bash -c "ln -s /usr/bin/python3 /usr/bin/python"
+
+	display_alert "Upgrading packages in" "${target_dir}" "info"
+	chroot "${target_dir}" /bin/bash -c "apt-get -q update; apt-get -q -y upgrade; apt-get clean"
+	date +%s >"$target_dir/root/.update-timestamp"
+
+	case $release in
+	bullseye|focal|hirsute|sid)
+		chroot "${target_dir}" /bin/bash -c "apt-get install python-is-python3"
+		;;
+	esac
+
 	touch "${target_dir}"/root/.debootstrap-complete
 	display_alert "Debootstrap complete" "${release}/${arch}" "info"
 } #############################################################################
@@ -147,27 +175,62 @@ chroot_build_packages()
 		for arch in $target_arch; do
 			display_alert "Starting package building process" "$release/$arch" "info"
 
-			local target_dir
-			target_dir="${SRC}/cache/buildpkg/${release}-${arch}-v${CHROOT_CACHE_VERSION}"
+			local t_name=${release}-${arch}-v${CHROOT_CACHE_VERSION}
 			local distcc_bindaddr="127.0.0.2"
 
-			[[ ! -f "${target_dir}"/root/.debootstrap-complete ]] && create_chroot "${target_dir}" "${release}" "${arch}"
-			[[ ! -f "${target_dir}"/root/.debootstrap-complete ]] && exit_with_error "Creating chroot failed" "${release}/${arch}"
+			# Create a clean environment archive if it does not exist.
+			if [ ! -f "${SRC}/cache/buildpkg/${t_name}.tar.xz" ]; then
+				local tmp_dir=$(mktemp -d "${SRC}"/.tmp/debootstrap-XXXXX)
+				create_chroot "${tmp_dir}/${t_name}" "${release}" "${arch}"
+				display_alert "Create a clean Environment archive" "${t_name}.tar.xz" "info"
+				(
+					tar -cp --directory="${tmp_dir}/" ${t_name} \
+					| pv -p -b -r -s "$(du -sb "${tmp_dir}/${t_name}" | cut -f1)" \
+					| pixz -4 >"${SRC}/cache/buildpkg/${t_name}.tar.xz"
+				)
+				rm -rf $tmp_dir
+			fi
+
+			# Unpack the clean environment archive, if it exists.
+			if [ -f "${SRC}/cache/buildpkg/${t_name}.tar.xz" ]; then
+				local tmp_dir=$(mktemp -d "${SRC}"/.tmp/build-XXXXX)
+				(	cd $tmp_dir
+					display_alert "Unpack the clean environment" "${t_name}.tar.xz" "info"
+					tar -xJf "${SRC}/cache/buildpkg/${t_name}.tar.xz" || \
+					exit_with_error "Is not extracted" "${SRC}/cache/buildpkg/${t_name}.tar.xz"
+				)
+				target_dir="$tmp_dir/${t_name}"
+			else
+				exit_with_error "Creating chroot failed" "${release}/${arch}"
+			fi
 
-			[[ -f /var/run/distcc/"${release}-${arch}".pid ]] && kill "$(<"/var/run/distcc/${release}-${arch}.pid")" > /dev/null 2>&1
+			[[ -f /var/run/distcc/"${release}-${arch}".pid ]] &&
+				kill "$(<"/var/run/distcc/${release}-${arch}.pid")" > /dev/null 2>&1
 
 			chroot_prepare_distccd "${release}" "${arch}"
 
 			# DISTCC_TCP_DEFER_ACCEPT=0
-			DISTCC_CMDLIST=/tmp/distcc/${release}-${arch}/cmdlist TMPDIR=/tmp/distcc distccd --daemon \
-				--pid-file "/var/run/distcc/${release}-${arch}.pid" --listen $distcc_bindaddr --allow 127.0.0.0/24 \
+			DISTCC_CMDLIST=/tmp/distcc/${release}-${arch}/cmdlist \
+				TMPDIR=/tmp/distcc distccd --daemon \
+				--pid-file "/var/run/distcc/${release}-${arch}.pid" \
+				--listen $distcc_bindaddr --allow 127.0.0.0/24 \
 				--log-file "/tmp/distcc/${release}-${arch}.log" --user distccd
 
+			[[ -d $target_dir ]] ||
+				exit_with_error "Clean Environment is not visible" "$target_dir"
+
 			local t=$target_dir/root/.update-timestamp
 			if [[ ! -f ${t} || $(( ($(date +%s) - $(<"${t}")) / 86400 )) -gt 7 ]]; then
 				display_alert "Upgrading packages" "$release/$arch" "info"
 				systemd-nspawn -a -q -D "${target_dir}" /bin/bash -c "apt-get -q update; apt-get -q -y upgrade; apt-get clean"
 				date +%s > "${t}"
+				display_alert "Repack a clean Environment archive after upgrading" "${t_name}.tar.xz" "info"
+				rm "${SRC}/cache/buildpkg/${t_name}.tar.xz"
+				(
+					tar -cp --directory="${tmp_dir}/" ${t_name} \
+					| pv -p -b -r -s "$(du -sb "${tmp_dir}/${t_name}" | cut -f1)" \
+					| pixz -4 >"${SRC}/cache/buildpkg/${t_name}.tar.xz"
+				)
 			fi
 
 			for plugin in "${SRC}"/packages/extras-buildpkgs/*.conf; do
@@ -200,12 +263,29 @@ chroot_build_packages()
 					display_alert "Packages are up to date" "$package_name $release/$arch" "info"
 					continue
 				fi
+
+				# Delete the environment if there was a build in it.
+				# And unpack the clean environment again.
+				if [[ -f "${target_dir}"/root/build.sh ]] && [[ -d $tmp_dir ]]; then
+					rm -rf $tmp_dir
+					local tmp_dir=$(mktemp -d "${SRC}"/.tmp/build-XXXXX)
+					(	cd $tmp_dir
+						display_alert "Unpack the clean environment" "${t_name}.tar.xz" "info"
+						tar -xJf "${SRC}/cache/buildpkg/${t_name}.tar.xz" || \
+						exit_with_error "Is not extracted" "${SRC}/cache/buildpkg/${t_name}.tar.xz"
+					)
+					target_dir="$tmp_dir/${t_name}"
+				fi
+
 				display_alert "Building packages" "$package_name $release/$arch" "ext"
+				ts=$(date +%s)
 				local dist_builddeps_name="package_builddeps_${release}"
 				[[ -v $dist_builddeps_name ]] && package_builddeps="${package_builddeps} ${!dist_builddeps_name}"
 
 				# create build script
+				LOG_OUTPUT_FILE=/root/build-"${package_name}".log
 				create_build_script
+				unset LOG_OUTPUT_FILE
 
 				fetch_from_repo "$package_repo" "extra/$package_name" "$package_ref"
 
@@ -223,7 +303,12 @@ chroot_build_packages()
 					built_ok+=("$package_name:$release/$arch")
 				fi
 				mv "${target_dir}"/root/*.deb "${plugin_target_dir}" 2>/dev/null
+				mv "${target_dir}"/root/*.log "$DEST/${LOG_SUBPATH}/"
+				te=$(date +%s)
+				display_alert "Build time $package_name " " $(($te - $ts)) sec." "info"
 			done
+			# Delete a temporary directory
+			if [ -d $tmp_dir ]; then rm -rf $tmp_dir;fi
 			# cleanup for distcc
 			kill $(</var/run/distcc/${release}-${arch}.pid)
 		done
@@ -263,39 +348,27 @@ create_build_script ()
 	export DEBEMAIL="$MAINTAINERMAIL"
 	$(declare -f display_alert)
 
-	cd /root/build
-	if [[ -n "${package_builddeps}" ]]; then
-		# can be replaced with mk-build-deps
-		deps=()
-		installed=\$(
-			dpkg-query -W -f '\${db:Status-Abbrev}|\${binary:Package}\n' '*' 2>/dev/null | \
-			grep '^ii' | \
-			awk -F '|' '{print \$2}' | \
-			cut -d ':' -f 1
-		)
-
-		for packet in $package_builddeps
-		do
-			grep -q -x -e "\$packet" <<< "\$installed" || deps+=("\$packet")
-		done
-
-		if [[ \${#deps[@]} -gt 0 ]]; then
-			display_alert "Installing build dependencies"
-			apt-get -y -q update
-			apt-get -y -q \
-					--no-install-recommends \
-					--show-progress \
-					-o DPKG::Progress-Fancy=1 install "\${deps[@]}"
-		fi
-	fi
+	LOG_OUTPUT_FILE=$LOG_OUTPUT_FILE
+	$(declare -f install_pkg_deb)
 
+	cd /root/build
 	display_alert "Copying sources"
 	rsync -aq /root/sources/"${package_name}" /root/build/
 
 	cd /root/build/"${package_name}"
 	# copy overlay / "debianization" files
 	[[ -d "/root/overlay/${package_name}/" ]] && rsync -aq /root/overlay/"${package_name}" /root/build/
 
+	package_builddeps="$package_builddeps"
+	if [ -z "\$package_builddeps" ]; then
+		# Calculate build dependencies by a standard function and
+		# еxclude special comparison characters like "|" "(>= 9)"
+		package_builddeps="\$(dpkg-checkbuilddeps |& awk -F":" '{gsub(/[|]|[(].*[)]/, " ", \$0); print \$NF}')"
+	fi
+	if [[ -n "\${package_builddeps}" ]]; then
+		install_pkg_deb \${package_builddeps}
+	fi
+
 	# set upstream version
 	[[ -n "${package_upstream_version}" ]] && debchange --preserve --newversion "${package_upstream_version}" "Import from upstream"
 
@@ -304,7 +377,9 @@ create_build_script ()
 	debchange -l~armbian"${REVISION}"+ "Custom $VENDOR release"
 
 	display_alert "Building package"
-	dpkg-buildpackage -b -us -j2
+	# Set the number of build threads and certainly send
+	# the standard error stream to the log file.
+	dpkg-buildpackage -b -us -j${NCPU_CHROOT:-2} 2>>\$LOG_OUTPUT_FILE
 
 	if [[ \$? -eq 0 ]]; then
 		cd /root/build

diff --git a/lib/general.sh b/lib/general.sh
@@ -24,10 +24,13 @@
 # addtorepo
 # repo-remove-old-packages
 # wait_for_package_manager
+# install_pkg_deb
 # prepare_host_basic
 # prepare_host
 # webseed
 # download_and_verify
+# show_developer_warning
+# show_checklist_variables
 
 
 # cleaning <target>
@@ -1173,6 +1176,92 @@ wait_for_package_manager()
 
 
 
+# Installing debian packages in the armbian build system.
+#
+# list="upgrade clean pkg1 pkg2 pkg3 pkgbadname pkg-1.0 | pkg-2.0 pkg5 (>= 9)"
+# install_pkg_deb $list
+#
+# If the package has a bad name, we will see it in the log file.
+# If there is an LOG_OUTPUT_FILE variable and it has a value as
+# the full real path to the log file, then all the information will be there.
+#
+# The LOG_OUTPUT_FILE variable must be defined in the calling function
+# before calling the install_pkg_deb function and unset after.
+#
+install_pkg_deb ()
+{
+	local list=""
+	local log_file
+	local for_install
+	local need_upgrade=false
+	local need_clean=false
+	local _line=${BASH_LINENO[0]}
+	local _function=${FUNCNAME[1]}
+	local _file=$(basename "${BASH_SOURCE[1]}")
+	local tmp_file=$(mktemp /tmp/install_log_XXXXX)
+	export DEBIAN_FRONTEND=noninteractive
+
+	list=$(
+	for p in $*;do
+		case $p in
+			upgrade) need_upgrade=true; continue ;;
+			clean) need_clean=true; continue ;;
+			\||\(*|*\)) continue ;;
+		esac
+		echo " $p"
+	done
+	)
+
+	if [ -d $(dirname $LOG_OUTPUT_FILE) ]; then
+		log_file=${LOG_OUTPUT_FILE}
+	else
+		log_file="${SRC}/output/${LOG_SUBPATH}/install.log"
+	fi
+
+	echo -e "\nInstalling packages in function: $_function" "[$_file:$_line]" >>$log_file
+	echo -e "\nIncoming list:" >>$log_file
+	printf "%-30s %-30s %-30s %-30s\n" $list >>$log_file
+	echo "" >>$log_file
+
+	(	apt-get -qq update
+		if $need_upgrade; then apt-get -y upgrade;fi
+	) 2>>$log_file
+
+	# If the package is not installed, check the latest
+	# up-to-date version in the apt cache.
+	# Exclude bad package names and send a message to the log.
+	for_install=$(
+	for p in $list;do
+	  if $(dpkg-query -W -f '${db:Status-Abbrev}' $p |& awk '/ii/{exit 1}');then
+		apt-cache  show $p -o APT::Cache::AllVersions=no |& \
+		awk -v p=$p -v tmp_file=$tmp_file \
+		'/^Package:/{print $2} /^E:/{print "Bad package name: ",p >>tmp_file}'
+	  fi
+	done
+	)
+
+	# This information should be logged.
+	if [ -s $tmp_file ]; then
+		cat $tmp_file >>$log_file
+	fi
+
+	if [ -n "$for_install" ]; then
+
+		apt-get install -qq -y --no-install-recommends $for_install 2>>$log_file
+
+		# We will show the status after installation
+		echo -e "\nstatus after installation:" >>$log_file
+		dpkg-query -W \
+		  -f '${binary:Package;-27} ${Version;-23} [ ${Status} ]\n' \
+		  $for_install >>$log_file
+	fi
+
+	if $need_clean;then apt-get clean; fi
+	rm $tmp_file
+}
+
+
+
 # prepare_host_basic
 #
 # * installs only basic packages
@@ -1737,7 +1826,7 @@ show_checklist_variables ()
 	for var in $checklist;do
 		eval pval=\$$var
 		echo -e "\n$var =:" >>$log_file
-		if [ $(echo "$pval" | gawk -F"/" '{print NF}') -ge 4 ];then
+		if [ $(echo "$pval" | awk -F"/" '{print NF}') -ge 4 ];then
 			printf "%s\n" $pval >>$log_file
 		else
 			printf "%-30s %-30s %-30s %-30s\n" $pval >>$log_file

diff --git a/packages/extras-buildpkgs/90-hostapd-realtek.conf b/packages/extras-buildpkgs/90-hostapd-realtek.conf
@@ -3,10 +3,6 @@ local package_name="hostapd-realtek"
 local package_repo="http://w1.fi/hostap.git"
 local package_ref="tag:hostap_2_5"
 local package_upstream_version="3:2.5-4"
-local package_builddeps="pkg-config libreadline-dev libpcsclite-dev libnl-route-3-dev libnl-genl-3-dev libnl-3-dev libncurses5-dev libdbus-1-dev docbook-utils docbook-to-man"
-local package_builddeps_jessie="libssl-dev"
-local package_builddeps_xenial="libssl-dev"
-local package_builddeps_stretch="libssl1.0-dev"
 local package_install_target="hostapd-realtek"
 local package_component="${release}-utils"
 

diff --git a/packages/extras-buildpkgs/90-hostapd.conf b/packages/extras-buildpkgs/90-hostapd.conf
@@ -3,7 +3,6 @@ local package_name="hostapd"
 local package_repo="http://w1.fi/hostap.git"
 local package_ref="tag:hostap_2_9"
 local package_upstream_version="3:2.9-102"
-local package_builddeps="pkg-config libssl-dev libreadline-dev libpcsclite-dev libnl-route-3-dev libnl-genl-3-dev libnl-3-dev libncurses5-dev libdbus-1-dev docbook-utils docbook-to-man"
 local package_install_target="hostapd"
 local package_component="${release}-utils"
 

diff --git a/packages/extras-buildpkgs/90-sunxi-tools.conf b/packages/extras-buildpkgs/90-sunxi-tools.conf
@@ -3,7 +3,7 @@ local package_name="sunxi-tools"
 local package_repo="https://github.com/linux-sunxi/sunxi-tools.git"
 local package_ref="branch:master"
 local package_upstream_version="1.4.2-2"
-local package_builddeps="libusb-1.0-0-dev ruby binutils-arm-none-eabi pkg-config u-boot-tools zlib1g-dev libfdt-dev"
+
 local package_install_target="sunxi-tools"
 local package_component="${release}-utils"