diff --git a/config/templates/Dockerfile b/config/templates/Dockerfile index 8ed9e800b31b..814b9f2dacca 100644 --- a/config/templates/Dockerfile +++ b/config/templates/Dockerfile @@ -115,6 +115,7 @@ RUN apt-get update \ xfsprogs \ xxd \ zip \ + zstd \ zlib1g-dev \ && rm -rf /var/lib/apt/lists/* RUN locale-gen en_US.UTF-8 diff --git a/lib/configuration.sh b/lib/configuration.sh index a3fa1097b744..cd690ea44034 100644 --- a/lib/configuration.sh +++ b/lib/configuration.sh @@ -32,8 +32,8 @@ cd "${SRC}" || exit # if variable not provided, check which is current version in the cache storage if [[ -z "${ROOTFSCACHE_VERSION}" ]]; then - ROOTFSCACHE_VERSION=$(wget --tries=10 -O - -o /dev/null https://github.com/armbian/mirror/releases/download/rootfs/rootfscache.version || true) - ROOTFSCACHE_VERSION=${ROOTFSCACHE_VERSION:-"0"} + ROOTFSCACHE_VERSION=$(curl https://api.github.com/repos/armbian/cache/releases/latest -s | jq .tag_name -r || true) + ROOTFSCACHE_VERSION=${ROOTFSCACHE_VERSION:-"0003"} fi [[ -z "${CHROOT_CACHE_VERSION}" ]] && CHROOT_CACHE_VERSION=7 diff --git a/lib/debootstrap.sh b/lib/debootstrap.sh index 5f189d09fc8b..4822c1158dc1 100644 --- a/lib/debootstrap.sh +++ b/lib/debootstrap.sh @@ -120,35 +120,57 @@ PRE_INSTALL_DISTRIBUTION_SPECIFIC create_rootfs_cache() { + if [[ "$ROOT_FS_CREATE_ONLY" == "yes" ]]; then + local cycles=1 + else + local cycles=3 + fi + + INITAL_ROOTFSCACHE_VERSION=$ROOTFSCACHE_VERSION + + # seek last cache, proceed to previous otherwise build it + for ((n = 0; n < cycles; n++)); do + + ROOTFSCACHE_VERSION=$(expr $INITAL_ROOTFSCACHE_VERSION - $n) + ROOTFSCACHE_VERSION=$(printf "%04d\n" ${ROOTFSCACHE_VERSION}) local packages_hash=$(get_package_list_hash "$ROOTFSCACHE_VERSION") local cache_type="cli" [[ ${BUILD_DESKTOP} == yes ]] && local cache_type="xfce-desktop" [[ -n ${DESKTOP_ENVIRONMENT} ]] && local cache_type="${DESKTOP_ENVIRONMENT}" [[ ${BUILD_MINIMAL} == yes ]] && local cache_type="minimal" - local cache_name=${RELEASE}-${cache_type}-${ARCH}.$packages_hash.tar.lz4 + local cache_name=${RELEASE}-${cache_type}-${ARCH}.$packages_hash.tar.zst local cache_fname=${SRC}/cache/rootfs/${cache_name} - local display_name=${RELEASE}-${cache_type}-${ARCH}.${packages_hash:0:3}...${packages_hash:29}.tar.lz4 + local display_name=${RELEASE}-${cache_type}-${ARCH}.${packages_hash:0:3}...${packages_hash:29}.tar.zst + + [[ "$ROOT_FS_CREATE_ONLY" == yes ]] && break - if [[ -f ${cache_fname} && -f ${cache_fname}.aria2 && "$ROOT_FS_CREATE_ONLY" != "yes" ]]; then + if [[ -f ${cache_fname} && -f ${cache_fname}.aria2 ]]; then rm ${cache_fname}* display_alert "Partially downloaded file. Re-start." download_and_verify "_rootfs" "$cache_name" fi - if [[ -f ${cache_fname} && "$ROOT_FS_CREATE_ONLY" == "yes" ]]; then - echo "$cache_fname" > $cache_fname.current - display_alert "Checking cache integrity" "$display_name" "info" - sudo lz4 -tqq ${cache_fname} - [[ $? -ne 0 ]] && rm $cache_fname && exit_with_error "Cache $cache_fname is corrupted and was deleted. Please restart!" - # sign if signature is missing - if [[ -n "${GPG_PASS}" && "${SUDO_USER}" && ! -f ${cache_fname}.asc ]]; then - [[ -n ${SUDO_USER} ]] && sudo chown -R ${SUDO_USER}:${SUDO_USER} "${DEST}"/images/ - echo "${GPG_PASS}" | sudo -H -u ${SUDO_USER} bash -c "gpg --passphrase-fd 0 --armor --detach-sign --pinentry-mode loopback --batch --yes ${cache_fname}" || exit 1 - fi - elif [[ "$ROOT_FS_CREATE_ONLY" != "yes" ]]; then + display_alert "Checking local cache" "$display_name" "info" + + if [[ -f $cache_fname ]]; then + break + else display_alert "searching on servers" download_and_verify "_rootfs" "$cache_name" + [[ -f ${cache_fname} ]] && break + fi + + if [[ ! -f $cache_fname ]]; then + display_alert "not found: try to use previous cache" + fi + done + + # check if cache exists and we want to make it + if [[ -f ${cache_fname} && "$ROOT_FS_CREATE_ONLY" == "yes" ]]; then + display_alert "Checking cache integrity" "$display_name" "info" + sudo zstd -tqq ${cache_fname} + [[ $? -ne 0 ]] && rm $cache_fname && exit_with_error "Cache $cache_fname is corrupted and was deleted. Please restart!" fi # if aria2 file exists download didn't succeeded @@ -156,7 +178,7 @@ create_rootfs_cache() local date_diff=$(( ($(date +%s) - $(stat -c %Y $cache_fname)) / 86400 )) display_alert "Extracting $display_name" "$date_diff days old" "info" - pv -p -b -r -c -N "[ .... ] $display_name" "$cache_fname" | lz4 -dc | tar xp --xattrs -C $SDCARD/ + pv -p -b -r -c -N "[ .... ] $display_name" "$cache_fname" | zstdmt -dc | tar xp --xattrs -C $SDCARD/ [[ $? -ne 0 ]] && rm $cache_fname && exit_with_error "Cache $cache_fname is corrupted and was deleted. Restart." rm $SDCARD/etc/resolv.conf echo "nameserver $NAMESERVER" >> $SDCARD/etc/resolv.conf @@ -362,7 +384,7 @@ create_rootfs_cache() umount_chroot "$SDCARD" tar cp --xattrs --directory=$SDCARD/ --exclude='./dev/*' --exclude='./proc/*' --exclude='./run/*' --exclude='./tmp/*' \ - --exclude='./sys/*' --exclude='./home/*' --exclude='./root/*' . | pv -p -b -r -s $(du -sb $SDCARD/ | cut -f1) -N "$display_name" | lz4 -5 -c > $cache_fname + --exclude='./sys/*' --exclude='./home/*' --exclude='./root/*' . | pv -p -b -r -s $(du -sb $SDCARD/ | cut -f1) -N "$display_name" | zstdmt -5 -c > $cache_fname # sign rootfs cache archive that it can be used for web cache once. Internal purposes if [[ -n "${GPG_PASS}" && "${SUDO_USER}" ]]; then diff --git a/lib/general.sh b/lib/general.sh index 2680129f7ff5..c31995b5985b 100644 --- a/lib/general.sh +++ b/lib/general.sh @@ -97,7 +97,7 @@ cleaning() ;; oldcache) # remove old `cache/rootfs` except for the newest 8 files - if [[ -d "${SRC}"/cache/rootfs && $(ls -1 "${SRC}"/cache/rootfs/*.lz4 2> /dev/null | wc -l) -gt "${ROOTFS_CACHE_MAX}" ]]; then + if [[ -d "${SRC}"/cache/rootfs && $(ls -1 "${SRC}"/cache/rootfs/*.zst* 2> /dev/null | wc -l) -gt "${ROOTFS_CACHE_MAX}" ]]; then display_alert "Cleaning" "rootfs cache (old)" "info" (cd "${SRC}"/cache/rootfs; ls -t *.lz4 | sed -e "1,${ROOTFS_CACHE_MAX}d" | xargs -d '\n' rm -f) # Remove signatures if they are present. We use them for internal purpose @@ -1406,7 +1406,7 @@ prepare_host() nfs-kernel-server ntpdate p7zip-full parted patchutils pigz pixz \ pkg-config pv python3-dev python3-distutils qemu-user-static rsync swig \ systemd-container u-boot-tools udev unzip uuid-dev wget whiptail zip \ - zlib1g-dev" + zlib1g-dev zstd" if [[ $(dpkg --print-architecture) == amd64 ]]; then @@ -1685,6 +1685,12 @@ download_and_verify() return fi + # rootfs has its own infra + if [[ "${remotedir}" == "_rootfs" ]]; then + local server="https://cache.armbian.com/" + remotedir="rootfs/$ROOTFSCACHE_VERSION" + fi + # switch to china mirror if US timeouts timeout 10 curl --location --head --fail --silent ${server}${remotedir}/${filename} 2>&1 >/dev/null if [[ $? -ne 7 && $? -ne 22 && $? -ne 0 ]]; then @@ -1699,12 +1705,6 @@ download_and_verify() fi fi - # rootfs has its own infra - if [[ "${remotedir}" == "_rootfs" ]]; then - local server="https://cache.armbian.com/" - remotedir="rootfs" - fi - # check if file exists on remote server before running aria2 downloader timeout 10 curl --location --head --fail --silent ${server}${remotedir}/${filename} 2>&1 >/dev/null [[ $? -ne 0 ]] && return