From f46d7d3d248db5467b4e3e052ec3363c5b16c510 Mon Sep 17 00:00:00 2001 From: MichaIng <28480705+MichaIng@users.noreply.github.com> Date: Sat, 6 Apr 2019 19:44:05 +0200 Subject: [PATCH 1/6] META | DietPi-Image: Create from disk or shrink an image file --- .meta/dietpi-image | 137 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 137 insertions(+) create mode 100644 .meta/dietpi-image diff --git a/.meta/dietpi-image b/.meta/dietpi-image new file mode 100644 index 0000000000..c249453a4c --- /dev/null +++ b/.meta/dietpi-image @@ -0,0 +1,137 @@ +#!/bin/bash +{ + #//////////////////////////////////// + # DietPi Image creation/finalise Script + # + #//////////////////////////////////// + # Created by Daniel Knight / daniel.knight@dietpi.com / dietpi.com + # Updated by MichaIng / micha@dietpi.com / dietpi.com + # + #//////////////////////////////////// + # + # Usage: + # - ./dietpi-image [GPT] + # source: /dev/* | *.img + # name: DietPi_-- + # device: RPi | OdroidXU4 | NativePC-BIOS | ... + # arch: ARMv6 | ARMv7 | ARMv8 | x86_64 + # distro: Stretch | Buster + # root_partition: 0 | 1 | 2 | ... ("0" means no partition table) + # "GPT": Required to correctly handle GPT partition tables + #//////////////////////////////////// + + # Grab input + SOURCE=$1 + NAME=$2 + ROOT=$3 + [[ $4 == GPT ]] && GPT=1 || GPT=0 + + # Check source + SOURCE_IS_FILE=1 + if [[ $SOURCE == /dev/* ]]; then + + SOURCE_IS_FILE=0 + [[ -b $SOURCE ]] || { echo "ERROR: Source block device ($SOURCE) does not exist"; exit 1; } + + else + + [[ -f $SOURCE ]] || { echo "ERROR: Source image file ($SOURCE) does not exist"; exit 1; } + + fi + + apt-get -y install gparted gdisk gpart dosfstools zerofree p7zip + + # GPT images (RockPro64) after reading the image AND again after shrinking + # To fix: + # - GPT PMBR size mismatch (4458495 != 15523839) will be corrected by w(rite). + # - 15523806 + # RUN + # - gdisk "$IMAGE_FP/$IMAGE_NAME" + # w | y + (( $GPT )) && gdisk $SOURCE + + # Attach .img file to loop device + if (( $SOURCE_IS_FILE )); then + + modprobe loop + SOURCE_FILE=$SOURCE + SOURCE=$(losetup -f) + losetup $SOURCE "$SOURCE_FILE" + partprobe $SOURCE + + fi + + # Estimate root partition dev + ROOT_DEV=$SOURCE + if [[ $ROOT != 0 ]]; then + + [[ $SOURCE =~ /mmcblk || $SOURCE =~ /nvme || $SOURCE =~ /loop ]] && ROOT_DEV="${SOURCE}p${ROOT}" + [[ $SOURCE =~ /[sh]d[a-z] ]] && ROOT_DEV="${SOURCE}${ROOT}" + + fi + + # Fsck + e2fsck -f $ROOT_DEV + + # Run the shrinking multiple times as the size can be reduced every time a bid more + for i in {0..2} + do + + # Shrink root file system to minimum size + resize2fs -M $ROOT_DEV + read -p 'The minimum file system size above is as well the minimum for gparted, besides gparted shows a larger minimum already. +Press CTRL+C to exit, else gparted will now start to allow partition resizing' + + # gparted to resize boot partition to minimum + gparted $SOURCE + read -p 'If failed press CTRL+C to exit and prevent further action, else, press any key to continue...' + + sync + + # Override free space with zeros to purge removed data and allow further image/archive size reduction + zerofree -v $ROOT_DEV + + done + + # GPT images (RockPro64) after reading the image AND again after shrinking + # To fix: + # - GPT PMBR size mismatch (4458495 != 15523839) will be corrected by w(rite). + # - 15523806 + # RUN + # - gdisk "$IMAGE_FP/$IMAGE_NAME" + # w | y + (( $GPT )) && gdisk $SOURCE + + # Estimate used size + # - Without partition table, the whole raw disk space needs to be cloned + if [[ $ROOT == 0 ]]; then + + SIZE=$(lsblk -o SIZE --bytes $SOURCE) + + else + + # 64 bytes for secondary GPT + safety net + SIZE=$(( ( $(fdisk -l -o End $SOURCE | tail -1) + 1 ) * 512 )) + + fi + SIZE=$(( $SIZE + 512*256 )) + + # If source is an image file, truncate to used size + if (( $SOURCE_IS_FILE )); then + + umount $SOURCE + truncate --size=$SIZE "$SOURCE_FILE" + + # else create .img file from block device source via dd, stopping at block count that matches used size + else + + dd if=$SOURCE of=/root/$NAME.img bs=1M status=progress count=$(( $SIZE / 1024 / 1024 + 1 )) + read -p "Image file created: /root/$NAME.img" + umount $SOURCE + + fi + + #----------------------------------------------------------------------------------- + exit + #----------------------------------------------------------------------------------- +} From d18cb70af8f184b03697e780e1753c06ec63d60c Mon Sep 17 00:00:00 2001 From: MichaIng <28480705+MichaIng@users.noreply.github.com> Date: Tue, 14 May 2019 15:58:41 +0200 Subject: [PATCH 2/6] META | DietPi-Image: Only resize2fs needs to run multiple times; Add root check, GB locale and /tmp working dir --- .meta/dietpi-image | 46 +++++++++++++++++++++++++++++++++------------- 1 file changed, 33 insertions(+), 13 deletions(-) diff --git a/.meta/dietpi-image b/.meta/dietpi-image index c249453a4c..50d0aaecad 100644 --- a/.meta/dietpi-image +++ b/.meta/dietpi-image @@ -26,6 +26,15 @@ ROOT=$3 [[ $4 == GPT ]] && GPT=1 || GPT=0 + # Check root + (( $UID )) && { echo 'ERROR: This script must run with root permissions. Please retry with "sudo".'; exit 1; } + + # Force locale + export LC_ALL='en_GB.UTF-8' + + # Move to /tmp for temp file creation + cd /tmp + # Check source SOURCE_IS_FILE=1 if [[ $SOURCE == /dev/* ]]; then @@ -39,6 +48,7 @@ fi + # Install required packages apt-get -y install gparted gdisk gpart dosfstools zerofree p7zip # GPT images (RockPro64) after reading the image AND again after shrinking @@ -73,26 +83,36 @@ # Fsck e2fsck -f $ROOT_DEV - # Run the shrinking multiple times as the size can be reduced every time a bid more - for i in {0..2} + # Shrink file system + # - Run multiple times until no change is done any more + out='' + FS_SIZE=0 + while : do - # Shrink root file system to minimum size - resize2fs -M $ROOT_DEV - read -p 'The minimum file system size above is as well the minimum for gparted, besides gparted shows a larger minimum already. -Press CTRL+C to exit, else gparted will now start to allow partition resizing' - - # gparted to resize boot partition to minimum - gparted $SOURCE - read -p 'If failed press CTRL+C to exit and prevent further action, else, press any key to continue...' + resize2fs -M $ROOT_DEV | tee resize2fs_out + if out=$(grep -m1 'Nothing to do!' resize2fs_out); then - sync + FS_SIZE=$(mawk '{print $5 * 4096}' <<< $out) + echo "[ OK ] Shrunken file system to $FS_SIZE bytes" + break - # Override free space with zeros to purge removed data and allow further image/archive size reduction - zerofree -v $ROOT_DEV + fi done + read -p 'The minimum file system size above is as well the minimum for gparted, besides gparted shows a larger minimum already. +Press CTRL+C to exit, else gparted will now start to allow partition resizing' + + # gparted to resize boot partition to minimum + gparted $SOURCE + read -p 'If failed press CTRL+C to exit and prevent further action, else, press any key to continue...' + + sync + + # Override free space with zeros to purge removed data and allow further image/archive size reduction + zerofree -v $ROOT_DEV + # GPT images (RockPro64) after reading the image AND again after shrinking # To fix: # - GPT PMBR size mismatch (4458495 != 15523839) will be corrected by w(rite). From a939be9731d7224cc633b2a2ac63e6bb3c188b05 Mon Sep 17 00:00:00 2001 From: MichaIng <28480705+MichaIng@users.noreply.github.com> Date: Tue, 14 May 2019 21:36:59 +0200 Subject: [PATCH 3/6] META | DietPi-Image: Mark partition resize non-interactively via parted; Create hashes + 7z archive --- .meta/dietpi-image | 49 ++++++++++++++++++++++++++++++---------------- 1 file changed, 32 insertions(+), 17 deletions(-) diff --git a/.meta/dietpi-image b/.meta/dietpi-image index 50d0aaecad..e52161bbb5 100644 --- a/.meta/dietpi-image +++ b/.meta/dietpi-image @@ -49,7 +49,7 @@ fi # Install required packages - apt-get -y install gparted gdisk gpart dosfstools zerofree p7zip + apt-get -y install gdist dosfstools parted zerofree p7zip # GPT images (RockPro64) after reading the image AND again after shrinking # To fix: @@ -83,7 +83,7 @@ # Fsck e2fsck -f $ROOT_DEV - # Shrink file system + # Shrink file system to minimum # - Run multiple times until no change is done any more out='' FS_SIZE=0 @@ -93,21 +93,22 @@ resize2fs -M $ROOT_DEV | tee resize2fs_out if out=$(grep -m1 'Nothing to do!' resize2fs_out); then - FS_SIZE=$(mawk '{print $5 * 4096}' <<< $out) - echo "[ OK ] Shrunken file system to $FS_SIZE bytes" + FS_SIZE=$(mawk '{print $5 * 8}' <<< $out) # 4k blocks => 512 byte sectors + echo "[ OK ] Shrunken root file system to $(( $FS_SIZE / 2048 + 1 )) MiB" break fi done - read -p 'The minimum file system size above is as well the minimum for gparted, besides gparted shows a larger minimum already. -Press CTRL+C to exit, else gparted will now start to allow partition resizing' + # Estimate minimum end sector + PART_START=$(fdisk -l -o Start $SOURCE | tail -1) # 512 byte sectors + PART_END=$(( $PART_START + $FS_SIZE )) - # gparted to resize boot partition to minimum - gparted $SOURCE - read -p 'If failed press CTRL+C to exit and prevent further action, else, press any key to continue...' + # Resize partition to minimum + parted $SOURCE unit s resizepart 2 $PART_END + # Sync changes to disk now sync # Override free space with zeros to purge removed data and allow further image/archive size reduction @@ -126,31 +127,45 @@ Press CTRL+C to exit, else gparted will now start to allow partition resizing' # - Without partition table, the whole raw disk space needs to be cloned if [[ $ROOT == 0 ]]; then - SIZE=$(lsblk -o SIZE --bytes $SOURCE) + PART_SIZE=$(lsblk -o SIZE --bytes $SOURCE) else - # 64 bytes for secondary GPT + safety net - SIZE=$(( ( $(fdisk -l -o End $SOURCE | tail -1) + 1 ) * 512 )) + PART_SIZE=$(( ( $(fdisk -l -o End $SOURCE | tail -1) + 1 ) * 512 )) fi - SIZE=$(( $SIZE + 512*256 )) + IMAGE_SIZE=$(( $PART_SIZE + 512*256 )) # 64 bytes for secondary GPT + safety net + + # Create final image in /root + cd /root # If source is an image file, truncate to used size if (( $SOURCE_IS_FILE )); then - umount $SOURCE - truncate --size=$SIZE "$SOURCE_FILE" + truncate --size=$IMAGE_SIZE "$SOURCE_FILE" # else create .img file from block device source via dd, stopping at block count that matches used size else - dd if=$SOURCE of=/root/$NAME.img bs=1M status=progress count=$(( $SIZE / 1024 / 1024 + 1 )) + dd if=$SOURCE of=$NAME.img bs=1M status=progress count=$(( $IMAGE_SIZE / 1024 / 1024 + 1 )) read -p "Image file created: /root/$NAME.img" - umount $SOURCE fi + # Generate hashes: MD5, SHA1, SHA256 + cat << _EOF_ > hash.txt +MD5: $(md5sum $NAME.img | mawk '{print $1}') +SHA1: $(sha1sum $NAME.img | mawk '{print $1}') +SHA256: $(sha256sum $NAME.img | mawk '{print $1}') +_EOF_ + + # Download current README + wget https://raw.githubusercontent.com/MichaIng/DietPi/master/README.md + + # Generate 7z archive + # NB: LZMA2 ultra compression method requires 2G RAM + 7zr a -m0=lzma2 -mx=9 $NAME.7z $NAME.img hash.txt README.md + #----------------------------------------------------------------------------------- exit #----------------------------------------------------------------------------------- From b27207b3692e72f03957e96fe2bae6ef71632e7e Mon Sep 17 00:00:00 2001 From: MichaIng <28480705+MichaIng@users.noreply.github.com> Date: Wed, 15 May 2019 01:01:21 +0200 Subject: [PATCH 4/6] v6.23 + DietPi-Image | Minor fixes and argument reorder --- .meta/dietpi-image | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.meta/dietpi-image b/.meta/dietpi-image index e52161bbb5..c2b5b4bf31 100644 --- a/.meta/dietpi-image +++ b/.meta/dietpi-image @@ -10,20 +10,20 @@ #//////////////////////////////////// # # Usage: - # - ./dietpi-image [GPT] + # - ./dietpi-image [GPT] # source: /dev/* | *.img + # root_partition: 0 | 1 | 2 | ... ("0" means no partition table) # name: DietPi_-- # device: RPi | OdroidXU4 | NativePC-BIOS | ... # arch: ARMv6 | ARMv7 | ARMv8 | x86_64 # distro: Stretch | Buster - # root_partition: 0 | 1 | 2 | ... ("0" means no partition table) # "GPT": Required to correctly handle GPT partition tables #//////////////////////////////////// # Grab input SOURCE=$1 - NAME=$2 - ROOT=$3 + ROOT=$2 + NAME=$3 [[ $4 == GPT ]] && GPT=1 || GPT=0 # Check root @@ -49,7 +49,7 @@ fi # Install required packages - apt-get -y install gdist dosfstools parted zerofree p7zip + apt-get -qq install gdisk dosfstools parted zerofree p7zip # GPT images (RockPro64) after reading the image AND again after shrinking # To fix: @@ -90,7 +90,7 @@ while : do - resize2fs -M $ROOT_DEV | tee resize2fs_out + resize2fs -M $ROOT_DEV 2>&1 | tee resize2fs_out if out=$(grep -m1 'Nothing to do!' resize2fs_out); then FS_SIZE=$(mawk '{print $5 * 8}' <<< $out) # 4k blocks => 512 byte sectors @@ -106,7 +106,7 @@ PART_END=$(( $PART_START + $FS_SIZE )) # Resize partition to minimum - parted $SOURCE unit s resizepart 2 $PART_END + parted -s $SOURCE unit s resizepart 2 $PART_END # Sync changes to disk now sync @@ -148,7 +148,7 @@ else dd if=$SOURCE of=$NAME.img bs=1M status=progress count=$(( $IMAGE_SIZE / 1024 / 1024 + 1 )) - read -p "Image file created: /root/$NAME.img" + echo "Image file created: /root/$NAME.img" fi From 6ae409962c80a9a4cfa23a341d3b41cc217fa672 Mon Sep 17 00:00:00 2001 From: MichaIng <28480705+MichaIng@users.noreply.github.com> Date: Sat, 18 May 2019 00:45:25 +0200 Subject: [PATCH 5/6] META | DietPi-Image: Fix parted call: It asks to proceed when shrinking the partition and there is no way to pre-answer this question with "-s" + minor --- .meta/dietpi-image | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.meta/dietpi-image b/.meta/dietpi-image index c2b5b4bf31..5abb48d0ba 100644 --- a/.meta/dietpi-image +++ b/.meta/dietpi-image @@ -95,6 +95,7 @@ FS_SIZE=$(mawk '{print $5 * 8}' <<< $out) # 4k blocks => 512 byte sectors echo "[ OK ] Shrunken root file system to $(( $FS_SIZE / 2048 + 1 )) MiB" + rm resize2fs_out break fi @@ -106,11 +107,15 @@ PART_END=$(( $PART_START + $FS_SIZE )) # Resize partition to minimum - parted -s $SOURCE unit s resizepart 2 $PART_END + echo '[ INFO ] Shrinking partition. Please press "y" when asked.' + parted $SOURCE unit s resizepart 2 $PART_END # Sync changes to disk now sync + # Re-read partition layout + partprobe $SOURCE + # Override free space with zeros to purge removed data and allow further image/archive size reduction zerofree -v $ROOT_DEV @@ -160,7 +165,7 @@ SHA256: $(sha256sum $NAME.img | mawk '{print $1}') _EOF_ # Download current README - wget https://raw.githubusercontent.com/MichaIng/DietPi/master/README.md + wget https://raw.githubusercontent.com/MichaIng/DietPi/master/README.md -O README.md # Generate 7z archive # NB: LZMA2 ultra compression method requires 2G RAM From fd0d48161ef915d7a7c1a6ec34a68fdffffd7716 Mon Sep 17 00:00:00 2001 From: MichaIng <28480705+MichaIng@users.noreply.github.com> Date: Sat, 18 May 2019 16:59:20 +0200 Subject: [PATCH 6/6] META | DietPi-Image: Increase minimum fs size by 4M, which was required on Raspbian Buster to boot properly --- .meta/dietpi-image | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/.meta/dietpi-image b/.meta/dietpi-image index 5abb48d0ba..93f4381dfe 100644 --- a/.meta/dietpi-image +++ b/.meta/dietpi-image @@ -93,9 +93,12 @@ resize2fs -M $ROOT_DEV 2>&1 | tee resize2fs_out if out=$(grep -m1 'Nothing to do!' resize2fs_out); then - FS_SIZE=$(mawk '{print $5 * 8}' <<< $out) # 4k blocks => 512 byte sectors - echo "[ OK ] Shrunken root file system to $(( $FS_SIZE / 2048 + 1 )) MiB" + # Re-add 4M to be failsafe, was required on Raspbian Buster for successful boot + FS_SIZE=$(mawk '{print $5 + 1024}' <<< $out) # 4k blocks rm resize2fs_out + resize2fs $ROOT_DEV $FS_SIZE + echo "[ OK ] Shrunken root file system to $(( $FS_SIZE / 256 + 1 )) MiB" + FS_SIZE=$(( $FS_SIZE * 8 )) # 4k blocks => 512 byte sectors break fi @@ -132,14 +135,14 @@ # - Without partition table, the whole raw disk space needs to be cloned if [[ $ROOT == 0 ]]; then - PART_SIZE=$(lsblk -o SIZE --bytes $SOURCE) + PART_SIZE=$(lsblk -o SIZE --bytes $SOURCE) # Byte else - PART_SIZE=$(( ( $(fdisk -l -o End $SOURCE | tail -1) + 1 ) * 512 )) + PART_SIZE=$(( ( $(fdisk -l -o End $SOURCE | tail -1) + 1 ) * 512 )) # 512 byte sectors => Byte fi - IMAGE_SIZE=$(( $PART_SIZE + 512*256 )) # 64 bytes for secondary GPT + safety net + IMAGE_SIZE=$(( $PART_SIZE + 512*256 )) # 64 byte for secondary GPT + safety net # Create final image in /root cd /root @@ -152,6 +155,7 @@ # else create .img file from block device source via dd, stopping at block count that matches used size else + [[ -f $NAME.img ]] && rm $NAME.img dd if=$SOURCE of=$NAME.img bs=1M status=progress count=$(( $IMAGE_SIZE / 1024 / 1024 + 1 )) echo "Image file created: /root/$NAME.img" @@ -169,6 +173,7 @@ _EOF_ # Generate 7z archive # NB: LZMA2 ultra compression method requires 2G RAM + [[ -f $NAME.7z ]] && rm $NAME.7z 7zr a -m0=lzma2 -mx=9 $NAME.7z $NAME.img hash.txt README.md #-----------------------------------------------------------------------------------