Rootfs cache refactoring (#4057)

* First part of rootfs cache refactoring

* Add support for checking previous caches.

* New default starting number

* Move back to n

* Add zstd compressor to the host dependencies

* Change cache compression to ZST

* Cleaning cache support

config/templates/Dockerfile

@@ -115,6 +115,7 @@ RUN apt-get update \
        xfsprogs \
        xxd \
        zip \
+       zstd \
        zlib1g-dev \
     && rm -rf /var/lib/apt/lists/*
 RUN locale-gen en_US.UTF-8

lib/configuration.sh

@@ -32,8 +32,8 @@ cd "${SRC}" || exit
 
 # if variable not provided, check which is current version in the cache storage
 if [[ -z "${ROOTFSCACHE_VERSION}" ]]; then
-	ROOTFSCACHE_VERSION=$(wget --tries=10 -O - -o /dev/null https://github.com/armbian/mirror/releases/download/rootfs/rootfscache.version || true)
-	ROOTFSCACHE_VERSION=${ROOTFSCACHE_VERSION:-"0"}
+	ROOTFSCACHE_VERSION=$(curl https://api.github.com/repos/armbian/cache/releases/latest -s | jq .tag_name -r || true)
+	ROOTFSCACHE_VERSION=${ROOTFSCACHE_VERSION:-"0003"}
 fi
 
 [[ -z "${CHROOT_CACHE_VERSION}" ]] && CHROOT_CACHE_VERSION=7

lib/debootstrap.sh

@@ -120,43 +120,65 @@ PRE_INSTALL_DISTRIBUTION_SPECIFIC
 create_rootfs_cache()
 {
 
+	if [[ "$ROOT_FS_CREATE_ONLY" == "yes" ]]; then
+		local cycles=1
+	else
+		local cycles=3
+	fi
+
+	INITAL_ROOTFSCACHE_VERSION=$ROOTFSCACHE_VERSION
+
+	# seek last cache, proceed to previous otherwise build it
+	for ((n = 0; n < cycles; n++)); do
+
+	ROOTFSCACHE_VERSION=$(expr $INITAL_ROOTFSCACHE_VERSION - $n)
+	ROOTFSCACHE_VERSION=$(printf "%04d\n" ${ROOTFSCACHE_VERSION})
 
 	local packages_hash=$(get_package_list_hash "$ROOTFSCACHE_VERSION")
 	local cache_type="cli"
 	[[ ${BUILD_DESKTOP} == yes ]] && local cache_type="xfce-desktop"
 	[[ -n ${DESKTOP_ENVIRONMENT} ]] && local cache_type="${DESKTOP_ENVIRONMENT}"
 	[[ ${BUILD_MINIMAL} == yes ]] && local cache_type="minimal"
-	local cache_name=${RELEASE}-${cache_type}-${ARCH}.$packages_hash.tar.lz4
+	local cache_name=${RELEASE}-${cache_type}-${ARCH}.$packages_hash.tar.zst
 	local cache_fname=${SRC}/cache/rootfs/${cache_name}
-	local display_name=${RELEASE}-${cache_type}-${ARCH}.${packages_hash:0:3}...${packages_hash:29}.tar.lz4
+	local display_name=${RELEASE}-${cache_type}-${ARCH}.${packages_hash:0:3}...${packages_hash:29}.tar.zst
+
+	[[ "$ROOT_FS_CREATE_ONLY" == yes ]] && break
 
-	if [[ -f ${cache_fname} && -f ${cache_fname}.aria2 && "$ROOT_FS_CREATE_ONLY" != "yes" ]]; then
+	if [[ -f ${cache_fname} && -f ${cache_fname}.aria2 ]]; then
 		rm ${cache_fname}*
 		display_alert "Partially downloaded file. Re-start."
 		download_and_verify "_rootfs" "$cache_name"
 	fi
 
-	if [[ -f ${cache_fname} && "$ROOT_FS_CREATE_ONLY" == "yes" ]]; then
-		echo "$cache_fname" > $cache_fname.current
-		display_alert "Checking cache integrity" "$display_name" "info"
-		sudo lz4 -tqq ${cache_fname}
-		[[ $? -ne 0 ]] && rm $cache_fname && exit_with_error "Cache $cache_fname is corrupted and was deleted. Please restart!"
-		# sign if signature is missing
-		if [[ -n "${GPG_PASS}" && "${SUDO_USER}" && ! -f ${cache_fname}.asc ]]; then
-			[[ -n ${SUDO_USER} ]] && sudo chown -R ${SUDO_USER}:${SUDO_USER} "${DEST}"/images/
-			echo "${GPG_PASS}" | sudo -H -u ${SUDO_USER} bash -c "gpg --passphrase-fd 0 --armor --detach-sign --pinentry-mode loopback --batch --yes ${cache_fname}" || exit 1
-		fi
-	elif [[ "$ROOT_FS_CREATE_ONLY" != "yes" ]]; then
+	display_alert "Checking local cache" "$display_name" "info"
+
+	if [[ -f $cache_fname ]]; then
+		break
+	else
 		display_alert "searching on servers"
 		download_and_verify "_rootfs" "$cache_name"
+		[[ -f ${cache_fname} ]] && break
+	fi
+
+	if [[ ! -f $cache_fname ]]; then
+		display_alert "not found: try to use previous cache"
+	fi
+	done
+
+	# check if cache exists and we want to make it
+        if [[ -f ${cache_fname} && "$ROOT_FS_CREATE_ONLY" == "yes" ]]; then
+                display_alert "Checking cache integrity" "$display_name" "info"
+                sudo zstd -tqq ${cache_fname}
+                [[ $? -ne 0 ]] && rm $cache_fname && exit_with_error "Cache $cache_fname is corrupted and was deleted. Please restart!"
 	fi
 
 	# if aria2 file exists download didn't succeeded
 	if [[ -f $cache_fname && ! -f $cache_fname.aria2 ]]; then
 
 		local date_diff=$(( ($(date +%s) - $(stat -c %Y $cache_fname)) / 86400 ))
 		display_alert "Extracting $display_name" "$date_diff days old" "info"
-		pv -p -b -r -c -N "[ .... ] $display_name" "$cache_fname" | lz4 -dc | tar xp --xattrs -C $SDCARD/
+		pv -p -b -r -c -N "[ .... ] $display_name" "$cache_fname" | zstdmt -dc | tar xp --xattrs -C $SDCARD/
 		[[ $? -ne 0 ]] && rm $cache_fname && exit_with_error "Cache $cache_fname is corrupted and was deleted. Restart."
 		rm $SDCARD/etc/resolv.conf
 		echo "nameserver $NAMESERVER" >> $SDCARD/etc/resolv.conf
@@ -362,7 +384,7 @@ create_rootfs_cache()
 		umount_chroot "$SDCARD"
 
 		tar cp --xattrs --directory=$SDCARD/ --exclude='./dev/*' --exclude='./proc/*' --exclude='./run/*' --exclude='./tmp/*' \
-			--exclude='./sys/*' --exclude='./home/*' --exclude='./root/*' . | pv -p -b -r -s $(du -sb $SDCARD/ | cut -f1) -N "$display_name" | lz4 -5 -c > $cache_fname
+			--exclude='./sys/*' --exclude='./home/*' --exclude='./root/*' . | pv -p -b -r -s $(du -sb $SDCARD/ | cut -f1) -N "$display_name" | zstdmt -5 -c > $cache_fname
 
 		# sign rootfs cache archive that it can be used for web cache once. Internal purposes
 		if [[ -n "${GPG_PASS}" && "${SUDO_USER}" ]]; then

lib/general.sh

@@ -97,7 +97,7 @@ cleaning()
 		;;
 
 		oldcache) # remove old `cache/rootfs` except for the newest 8 files
-		if [[ -d "${SRC}"/cache/rootfs && $(ls -1 "${SRC}"/cache/rootfs/*.lz4 2> /dev/null | wc -l) -gt "${ROOTFS_CACHE_MAX}" ]]; then
+		if [[ -d "${SRC}"/cache/rootfs && $(ls -1 "${SRC}"/cache/rootfs/*.zst* 2> /dev/null | wc -l) -gt "${ROOTFS_CACHE_MAX}" ]]; then
 			display_alert "Cleaning" "rootfs cache (old)" "info"
 			(cd "${SRC}"/cache/rootfs; ls -t *.lz4 | sed -e "1,${ROOTFS_CACHE_MAX}d" | xargs -d '\n' rm -f)
 			# Remove signatures if they are present. We use them for internal purpose
@@ -1406,7 +1406,7 @@ prepare_host()
 	nfs-kernel-server ntpdate p7zip-full parted patchutils pigz pixz          \
 	pkg-config pv python3-dev python3-distutils qemu-user-static rsync swig   \
 	systemd-container u-boot-tools udev unzip uuid-dev wget whiptail zip      \
-	zlib1g-dev"
+	zlib1g-dev zstd"
 
   if [[ $(dpkg --print-architecture) == amd64 ]]; then
 
@@ -1685,6 +1685,12 @@ download_and_verify()
 		return
 	fi
 
+	# rootfs has its own infra
+	if [[ "${remotedir}" == "_rootfs" ]]; then
+		local server="https://cache.armbian.com/"
+		remotedir="rootfs/$ROOTFSCACHE_VERSION"
+	fi
+
 	# switch to china mirror if US timeouts
 	timeout 10 curl --location --head --fail --silent ${server}${remotedir}/${filename} 2>&1 >/dev/null
 	if [[ $? -ne 7 && $? -ne 22 && $? -ne 0 ]]; then
@@ -1699,12 +1705,6 @@ download_and_verify()
 		fi
 	fi
 
-	# rootfs has its own infra
-	if [[ "${remotedir}" == "_rootfs" ]]; then
-		local server="https://cache.armbian.com/"
-		remotedir="rootfs"
-	fi
-
 	# check if file exists on remote server before running aria2 downloader
 	timeout 10 curl --location --head --fail --silent ${server}${remotedir}/${filename} 2>&1 >/dev/null
 	[[ $? -ne 0 ]] && return