Permalink
Switch branches/tags
Find file Copy path
Fetching contributors…
Cannot retrieve contributors at this time
executable file 1273 lines (1051 sloc) 39 KB
#!/bin/bash
## Copyright (C) 2012 - 2018 ENCRYPTED SUPPORT LP <adrelanos@riseup.net>
## See the file COPYING for copying conditions.
## x
[ -o xtrace ]
## returns:
## - 0, if -x is set
## - 1, if -x is not set
MINUS_X_SET="$?"
set -x
set -e
true "INFO: Currently running script: $BASH_SOURCE $@"
MYDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
cd "$MYDIR"
cd ..
## whonix_build_one_parsed is read by help-steps/variables (help-steps/parse-cmd)
## and by help-steps/mount-raw and help-steps/unmount-raw.
export whonix_build_one_parsed="1"
export VMNAME="internalrun"
cd help-steps
source pre
source colors
source variables
script_help() {
echo "\
Required options:
--report reportfile
--tempfolder folder
And either one of the following:
--ova ovafile
--vdi vdifile
--qcow qcowfile
--qcow2 qcow2file
--raw rawfile
--root /path/to/folder
Optional options:
--topcomment comment
--endcomment comment
--errorcomment comment
--nodeltempfolder
Usage examples:
From Whonix Source Code Folder...
sudo ./help-steps/analyze_image --report ~/report1 --tempfolder ~/whonix_binary/report_temp1 --ova ~/Whonix-Gateway-7.4.3.ova
sudo ./help-steps/analyze_image --report ~/report2 --tempfolder ~/whonix_binary/report_temp2 --ova ~/whonix_binary/Whonix-Gateway-7.4.3.ova
sudo ./help-steps/analyze_image --report ~/report3 --tempfolder ~/whonix_binary/report_temp3 --root /
sudo ./help-steps/analyze_image --report ~/report4 --tempfolder ~/whonix_binary/report_temp4 --raw ~/whonix_binary/Whonix-Gateway.raw
sudo ./help-steps/analyze_image --report ~/report5 --tempfolder ~/whonix_binary/report_temp5 --qcow2 ~/Whonix-Gateway.qcow2
From ~/whonix_binary folder...
vbindiff ./report_temp1/manual_analysis_folder/Whonix-Gateway-7.4.3.raw.dd.1000000 ./report_temp2/manual_analysis_folder/Whonix-Gateway-7.4.3.raw.dd.1000000
See https://www.whonix.org/wiki/Verifiable_Builds
and https://www.whonix.org/wiki/Trust"
}
parse_cmd() {
## Thanks to:
## http://mywiki.wooledge.org/BashFAQ/035
if [ "$*" = "" ]; then
error "no option chosen. Use --help."
fi
## defaults
ova="0"
vmdk="0"
vdi="0"
qcow="0"
qcow2="0"
raw="0"
if [ "$ANON_BUILD_INSTALL_TO_ROOT" = "" ]; then
ANON_BUILD_INSTALL_TO_ROOT="0"
fi
while :
do
case $1 in
-h | --help | -\?)
script_help
exit 0
;;
-o | --ova)
ova="1"
vmdk="1"
vdi="1"
ova_file="$2"
to_analyze_file="$2"
shift 2
;;
-vmdk | --vmdk)
vmdk="1"
vdi="1"
vmdk_file="$2"
to_analyze_file="$2"
shift 2
;;
-q | --vdi)
vdi="1"
vdi_file="$2"
to_analyze_file="$2"
shift 2
;;
-q | --qcow)
qcow="1"
qcow_file="$2"
to_analyze_file="$2"
shift 2
;;
-q2 | --qcow2)
qcow2="1"
qcow2_file="$2"
to_analyze_file="$2"
shift 2
;;
-raw | --raw | -img | --img)
raw="1"
raw_file="$2"
to_analyze_file="$2"
shift 2
;;
-bm | --bare-metal | --install-to-root | -itr | --root)
ANON_BUILD_INSTALL_TO_ROOT="1"
mount_folder="$2"
if [ "$mount_folder" = "" ]; then
echo 'Mount folder must not be empty. Use for example --root "/".'
exit 1
fi
shift 2
;;
-r | --report)
report_file="$2"
shift 2
;;
-t | --tempfolder)
tempfolder="$2"
shift 2
;;
-n | --nodeltempfolder)
nodeltempfolder="1"
shift
;;
-c | --topcomment)
topcomment="$2"
shift 2
;;
-e | --endcomment)
endcomment="$2"
shift 2
;;
-f | --errorcomment)
errorcomment="$2"
shift 2
;;
--minimal)
minimal_report="true"
shift
;;
--)
shift
break
;;
-*)
error "unknown option: $1"
;;
*)
break
;;
esac
done
if [ "$report_file" = "" ]; then
echo "${red}${bold} ERROR: no report_file chosen! ${reset}"
exit 1
fi
if [ "$tempfolder" = "" ]; then
echo "${red}${bold} ERROR: no tempfolder chosen! ${reset}"
exit 1
fi
}
error_handler() {
local exit_code="$?"
local bash_command="$BASH_COMMAND"
## Re-enable xtrace, in case it was deactivated.
set -x
unmount_image
true "
${red}${bold}bash_command${reset}: $bash_command
${red}${bold}exit_code${reset}: $exit_code
"
if [ "$errorcomment" = "" ]; then
errorcomment="ERROR: Unfinished report! Error detected!"
fi
errorcomment="\
################################################################################
$errorcomment
bash_command: $bash_command
exit_code: $exit_code"
echo "$errorcomment" >> "$report_file"
exit 1
}
trap "error_handler" ERR INT TERM
unmount_image() {
if [ "$ANON_BUILD_INSTALL_TO_ROOT" = "1" ]; then
true "${bold}${cyan}INFO: Skipping $FUNCNAME, because ANON_BUILD_INSTALL_TO_ROOT is 1. ${reset}"
return 0
fi
true "${bold}${cyan}INFO: Unmounting raw image... ${reset}"
## WHONIX_BUILD_MOUNT_RAW_FILE us read by help-steps/mount-raw
export WHONIX_BUILD_MOUNT_RAW_FILE="$raw_file_short_link"
"$WHONIX_SOURCE_HELP_STEPS_FOLDER"/unmount-raw
#sync
#sleep 1 &
#wait "$!"
#sync
#local command_v_exit_code="0"
#command -v guestunmount >/dev/null || { command_v_exit_code="$?" ; true; };
#if [ "$command_v_exit_code" = "0" ]; then
#true "${bold}${cyan}INFO: guestunmount available, using it, ok... ${reset}"
#guestunmount "$mount_folder"
#else
## guestunmount is not available in Debian Wheezy. Only since Debian Jessie.
#true "${bold}${cyan}INFO: guestunmount not available, using \"fusermount -u\" instead, ok... ${reset}"
#fusermount -u "$mount_folder"
#fi
unlink "$raw_file_short_link" || true
sync
true "${bold}${cyan}INFO: Unmounted raw image. ${reset}"
}
preparation() {
root_check
if [ "$minimal_report" = "true" ]; then
true "${cyan}INFO: Only creating minimal report file because minimal_report is true.${reset}"
rm --force "$report_file"
echo "$topcomment" >> "$report_file"
echo "Only creating minimal report file because minimal_report is true." >> "$report_file"
echo "$endcomment" >> "$report_file"
exit 0
fi
true "${cyan}INFO: user_name is set to $user_name ${reset}"
true "${cyan}INFO: Benchmarking \"sudo -u \"$user_name\" echo \"This is a test echo.\"\" using \"time\"... ${reset}"
time sudo $SUDO_OPTS echo "This is a test echo."
## {{ Sanity Tests.
local tool
local tools
tools="
readlink
sha512sum
dirname
basename
pwd
sudo
echo
mkdir
touch
command
rm
cp
chown
dd
sync
find
sort
stat
unlink
ln
"
for tool in $tools; do
command -v "$tool" >/dev/null
done
unset tool
unset tools
if [ "$ANON_BUILD_INSTALL_TO_ROOT" = "1" ]; then
## Tools for VM mouting not installed (not required) in this case.
true
else
tools="
guestmount
guestfish
virt-filesystems
"
for tool in $tools; do
command -v "$tool" >/dev/null
done
unset tool
unset tools
fi
## }}
rm --force "$report_file"
sync
## The WHONIX_BINARY folder should already be created. Just make sure it
## really is to allow running this script in non-Whonix environments as
## well.
sudo $SUDO_OPTS mkdir --parents "$WHONIX_BINARY"
sudo $SUDO_OPTS touch "$report_file"
sync
if [ ! "$nodeltempfolder" = "1" ]; then
true "${cyan} INFO: --nodeltempfolder was not used. Deleting $tempfolder... ${reset}"
rm --recursive --force "$tempfolder"
fi
## Using export, because mount_folder gets read by help-steps/mount-raw and
## help-steps/unmount-raw.
if [ "$ANON_BUILD_INSTALL_TO_ROOT" = "1" ]; then
## mount_folder has been set in parse_cmd
## Remove trailing slash so for example "/" becomes "".
mount_folder="${mount_folder%/}"
export mount_folder
else
export mount_folder="$tempfolder/mount_folder"
fi
extracted_ova_folder="$tempfolder/extracted_ova_folder"
vdi_folder="$tempfolder/vdi_folder"
qcow_folder="$tempfolder/qcow_folder"
qcow2_folder="$tempfolder/qcow2_folder"
raw_folder="$tempfolder/raw_folder"
auto_hash_folder="$tempfolder/auto_hash_folder"
initrd_folder="$tempfolder/initrd_folder"
extracted_initrd_folder="$tempfolder/extracted_initrd_folder"
debug_folder="$tempfolder/debug_folder"
manual_analysis_folder="$tempfolder/manual_analysis_folder"
to_analyze_file_basename="$(basename "$to_analyze_file")"
file_type_filename_without_extension="${to_analyze_file_basename%.*}"
if [ "$vmdk_file" = "" ]; then
vmdk_file="$extracted_ova_folder/$file_type_filename_without_extension-disk1.vmdk"
fi
if [ "$vdi_file" = "" ]; then
vdi_file="$vdi_folder/$file_type_filename_without_extension.vdi"
fi
if [ "$qcow_file" = "" ]; then
qcow_file="$qcow_folder/$file_type_filename_without_extension.qcow"
fi
if [ "$qcow2_file" = "" ]; then
qcow2_file="$qcow2_folder/$file_type_filename_without_extension.qcow2"
fi
if [ "$raw_file" = "" ]; then
raw_file="$raw_folder/$file_type_filename_without_extension.raw"
fi
raw_file_short_link="$WHONIX_BINARY/temp_short_link.raw"
vmdk_file_basename="$(basename "$vmdk_file")"
vdi_file_basename="$(basename "$vdi_file")"
qcow_file_basename="$(basename "$qcow_file")"
qcow2_file_basename="$(basename "$qcow2_file")"
raw_file_basename="$(basename "$raw_file")"
sudo $SUDO_OPTS mkdir --parents "$extracted_ova_folder"
sudo $SUDO_OPTS mkdir --parents "$mount_folder" || true
sudo $SUDO_OPTS mkdir --parents "$vdi_folder"
sudo $SUDO_OPTS mkdir --parents "$qcow_folder"
sudo $SUDO_OPTS mkdir --parents "$qcow2_folder"
sudo $SUDO_OPTS mkdir --parents "$raw_folder"
sudo $SUDO_OPTS mkdir --parents "$auto_hash_folder"
sudo $SUDO_OPTS mkdir --parents "$initrd_folder"
sudo $SUDO_OPTS mkdir --parents "$extracted_initrd_folder"
sudo $SUDO_OPTS mkdir --parents "$debug_folder"
sudo $SUDO_OPTS mkdir --parents "$manual_analysis_folder"
file_list_file="$auto_hash_folder/file_list"
rm --force "$file_list_file"
sudo $SUDO_OPTS touch "$file_list_file"
sync
}
parse_topcomment() {
if [ "$topcomment" = "" ]; then
topcomment="No topcomment."
fi
topcomment="\
$topcomment
################################################################################"
echo "$topcomment" >> "$report_file"
}
extract_ova() {
if [ "$ova" = "0" ]; then
true "${bold}${cyan}INFO: Skipping $FUNCNAME, because no ova chosen. ${reset}"
return 0
fi
if [ "$ANON_BUILD_INSTALL_TO_ROOT" = "1" ]; then
true "${bold}${cyan}INFO: Skipping $FUNCNAME, because ANON_BUILD_INSTALL_TO_ROOT is 1. ${reset}"
return 0
fi
cd "$extracted_ova_folder"
if [ -f "$vdi_file" ]; then
true "${bold}${cyan}INFO: Unpacking .ova not required, .vdi already exists, skipping. ${reset}"
else
if [ ! -f "$ova_file" ]; then
error "${red}${bold}ERROR: $ova_file does not exist. ${reset}"
else
true "${bold}${cyan}INFO: Unpacking ova: $ova_file... (This can take a while.) ${reset}"
sudo $SUDO_OPTS tar -xvf "$ova_file"
true "${bold}${cyan}INFO: Unpacked ova_file. ${reset}"
fi
fi
}
convert_vmdk_to_vdi() {
if [ "$vmdk" = "0" ]; then
true "${bold}${cyan}INFO: Skipping $FUNCNAME, because no vmdk chosen. ${reset}"
return 0
fi
if [ "$ANON_BUILD_INSTALL_TO_ROOT" = "1" ]; then
true "${bold}${cyan}INFO: Skipping $FUNCNAME, because ANON_BUILD_INSTALL_TO_ROOT is 1. ${reset}"
return 0
fi
if [ ! -f "$vmdk_file" ]; then
error "${red}${bold}ERROR: vmdk_file: $vmdk_file does not exist. ${reset}"
fi
if [ -f "$vdi_file" ]; then
true "${bold}${cyan}INFO: Converting vmdk to vdi not required, already done, skipping. ${reset}"
else
## Convert .vmdk to .vdi, since there is no Free Software for mounting .vmdk using command line.
true "${bold}${cyan}INFO: Converting vmdk to vdi... (This can take a while.) ${reset}"
## qemu-img version 1.6.1 fails with:
## qemu-img: 'image' uses a vmdk feature which is not supported by this qemu version: VMDK version 3
## https://bugs.launchpad.net/qemu/+bug/1253465
#sudo $SUDO_OPTS qemu-img convert "$vmdk_file" -O raw "$vdi_file"
## Debugging.
sudo $SUDO_OPTS qemu-img info "$vmdk_file"
sudo $SUDO_OPTS VBoxManage clonehd --format VDI "$vmdk_file" "$vdi_file"
## Debugging.
sudo $SUDO_OPTS qemu-img info "$vdi_file"
true "${bold}${cyan}INFO: Converted vmdk to vdi. ${reset}"
fi
}
convert_x_to_raw() {
if [ "$ANON_BUILD_INSTALL_TO_ROOT" = "1" ]; then
true "${bold}${cyan}INFO: Skipping $FUNCNAME, because ANON_BUILD_INSTALL_TO_ROOT is 1. ${reset}"
return 0
fi
if [ -f "$raw_file" ]; then
true "${bold}${cyan}INFO: Converting x to raw not required, already done, skipping. ${reset}"
else
if [ "$vdi" = "1" ]; then
image_type="vdi"
image_file="$vdi_file"
fi
if [ "$qcow" = "1" ]; then
image_type="qcow"
image_file="$qcow_file"
fi
if [ "$qcow2" = "1" ]; then
image_type="qcow2"
image_file="$qcow2_file"
fi
true "${bold}${cyan}INFO: Converting $image_type to raw... (This can take a while.) ${reset}"
## Debugging.
sudo $SUDO_OPTS qemu-img info "$image_file"
sudo $SUDO_OPTS qemu-img convert -p -O raw "$image_file" "$raw_file"
## Debugging.
sudo $SUDO_OPTS qemu-img info "$raw_file"
true "${bold}${cyan}INFO: Converted $image_type to raw. ${reset}"
fi
}
mount_image() {
if [ "$ANON_BUILD_INSTALL_TO_ROOT" = "1" ]; then
mount_raw_exit_code="0"
true "${bold}${cyan}INFO: Skipping $FUNCNAME, because ANON_BUILD_INSTALL_TO_ROOT is 1. ${reset}"
return 0
fi
true "${bold}${cyan}INFO: Mounting raw image... ${reset}"
## Workaround for a bug in kpartx, which fails to delete the loop device
## when using very long file names:
## https://www.redhat.com/archives/dm-devel/2014-July/msg00053.html
unlink "$raw_file_short_link" || true
sudo $SUDO_OPTS ln -s "$raw_file" "$raw_file_short_link"
## WHONIX_BUILD_MOUNT_RAW_FILE us read by help-steps/mount-raw
export WHONIX_BUILD_MOUNT_RAW_FILE="$raw_file_short_link"
mount_raw_exit_code="0"
"$WHONIX_SOURCE_HELP_STEPS_FOLDER"/mount-raw || { mount_raw_exit_code="$?" ; true; };
#sync
## Mounting read-only so the user or script can not accidentally delete
## files within the image.
#guestmount -o allow_other -a "$raw_file_short_link" -m /dev/sda1 --ro "$mount_folder"
#sync
if [ "$mount_raw_exit_code" = "0" ]; then
true "${bold}${cyan}INFO: Mounted raw image. ${reset}"
else
msg="WARNING: Mounting of raw image failed. This is expected for Whonix-Custom-Workstation."
true "${bold}${red}${msg}${reset}"
echo "${msg}" >> "$report_file"
fi
}
parse_file_system() {
if [ "$ANON_BUILD_INSTALL_TO_ROOT" = "1" ]; then
true "${bold}${cyan}INFO: Skipping $FUNCNAME, because ANON_BUILD_INSTALL_TO_ROOT is 1. ${reset}"
return 0
fi
true "${bold}${cyan}INFO: Parsing file systems... ${reset}"
if [ "$vdi" = "1" ]; then
virt-filesystems -a "$vdi_file" > "$auto_hash_folder/$vdi_file_basename.virt-filesystems-a" 2>&1
fi
if [ "$qcow" = "1" ]; then
virt-filesystems -a "$qcow_file" > "$auto_hash_folder/$qcow_file_basename.virt-filesystems-a" 2>&1
fi
if [ "$qcow2" = "1" ]; then
virt-filesystems -a "$qcow2_file" > "$auto_hash_folder/$qcow2_file_basename.virt-filesystems-a" 2>&1
fi
virt-filesystems -a "$raw_file" > "$auto_hash_folder/$raw_file_basename.virt-filesystems-a" 2>&1
true "${bold}${cyan}INFO: Parsed file systems. ${reset}"
}
parse_mbr() {
if [ "$ANON_BUILD_INSTALL_TO_ROOT" = "1" ]; then
true "${bold}${cyan}INFO: Skipping $FUNCNAME, because ANON_BUILD_INSTALL_TO_ROOT is 1. ${reset}"
return 0
fi
## For information about disk signatures, see:
## https://en.wikipedia.org/wiki/Master_boot_record
true "${bold}${cyan}INFO: Parsing MBR... ${reset}"
## Read ~1 MB at once.
dd if="$raw_file" of="$auto_hash_folder/$raw_file_basename.dd.0-1000000" bs=1000000 count=1
## Read byte by byte from 0 to 440.
## There should be no differences here.
dd if="$raw_file" of="$auto_hash_folder/$raw_file_basename.dd.0-440" bs=1 count=440
## Read byte by byte from 441 to 444.
## Disk signature may differ. (Fixed in Whonix 8.5.0.1 and above.)
dd if="$raw_file" of="$auto_hash_folder/$raw_file_basename.dd.441-444" bs=1 skip=440 count=3
## Read byte by byte from 445 to 1000000.
## There should be no differences here.
dd if="$raw_file" of="$auto_hash_folder/$raw_file_basename.dd.445-1000000" bs=1 skip=444 count=1000000
true "${bold}${cyan}INFO: Parsed MBR. ${reset}"
}
parse_vbr() {
if [ "$ANON_BUILD_INSTALL_TO_ROOT" = "1" ]; then
true "${bold}${cyan}INFO: Skipping $FUNCNAME, because ANON_BUILD_INSTALL_TO_ROOT is 1. ${reset}"
return 0
fi
true "${bold}${cyan}INFO: Parsing VBR... ${reset}"
if [ "$vdi" = "1" ]; then
guestfish --ro -a "$vdi_file" run : pread-device /dev/sda1 512 0 > "$auto_hash_folder/$vdi_file_basename.sda1_pread-device.512" 2>&1 || true
fi
if [ "$qcow" = "1" ]; then
guestfish --ro -a "$qcow_file" run : pread-device /dev/sda1 512 0 > "$auto_hash_folder/$qcow_file_basename.sda1_pread-device.512" 2>&1 || true
fi
if [ "$qcow2" = "1" ]; then
guestfish --ro -a "$qcow2_file" run : pread-device /dev/sda1 512 0 > "$auto_hash_folder/$qcow2_file_basename.sda1_pread-device.512" 2>&1 || true
fi
guestfish --ro -a "$raw_file" run : pread-device /dev/sda1 512 0 > "$auto_hash_folder/$raw_file_basename.sda1_pread-device.512" 2>&1 || true
true "${bold}${cyan}INFO: Parsed VBR. ${reset}"
}
cp_to_user() {
cp "$1" "$2"
chown --recursive "$user_name:$user_name" "$2"
}
parse_special_files() {
true "${bold}${cyan}INFO: Parsing special files... ${reset}"
###########################
## manual_analysis_folder #
###########################
cp_to_user "$mount_folder/etc/shadow" "$manual_analysis_folder/etc_shadow"
cp_to_user "$mount_folder/etc/shadow-" "$manual_analysis_folder/etc_shadow-"
## "|| true", because CI (Ubuntu) and custom builders may not use sysvinit.
cp_to_user "$mount_folder/etc/init.d/.depend.boot" "$manual_analysis_folder/etc_init.d_.depend.boot" || true
cp_to_user "$mount_folder/etc/init.d/.depend.start" "$manual_analysis_folder/etc_init.d_.depend.start" || true
cp_to_user "$mount_folder/etc/init.d/.depend.stop" "$manual_analysis_folder/etc_init.d_.depend.stop" || true
## {{{ /var/lib/initramfs-tools/
sudo $SUDO_OPTS mkdir --parents "$manual_analysis_folder/var_lib_initramfs-tools"
local file_name
shopt -s nullglob dotglob
for file_name in "$mount_folder/var/lib/initramfs-tools/"*; do
cp_to_user "$file_name" "$manual_analysis_folder/var_lib_initramfs-tools/"
done
unset file_name
shopt -u nullglob dotglob
## }}}
#################
## debug_folder #
#################
cp_to_user "$mount_folder/etc/fstab" "$debug_folder/etc_fstab"
## Legacy, old path to build version file.
## Allow running this script in non-Whonix environments as well.
if [ -e "$mount_folder/usr/share/whonix/build_version" ]; then
cp_to_user "$mount_folder/usr/share/whonix/build_version" "$debug_folder/usr_share_whonix_build_version"
fi
## Allow running this script in non-Whonix environments as well.
if [ -e "$mount_folder/var/lib/anon-dist/build_version" ]; then
cp_to_user "$mount_folder/var/lib/anon-dist/build_version" "$debug_folder/usr_lib_anon-dist_build_version"
echo "$debug_folder/usr_lib_anon-dist_build_version BEGIN..."
sudo $SUDO_OPTS cat "$debug_folder/usr_lib_anon-dist_build_version"
echo "END $debug_folder/usr_lib_anon-dist_build_version."
else
msg="Strange, $mount_folder/var/lib/anon-dist/build_version does not exist. Perhaps not running in Whonix."
echo "$msg" > "$debug_folder/usr_lib_anon-dist_build_version"
fi
cp_to_user "$mount_folder/etc/debootstrap/config" "$debug_folder/etc_debootstrap_config" || true
cp_to_user "$mount_folder/etc/debootstrap/stages/default_locales" "$debug_folder/etc_debootstrap_stages_default_locales" || true
## {{{ /usr/share/doc/whonix-*/changelog.Debian.gz
sudo $SUDO_OPTS mkdir --parents "$debug_folder/usr_share_doc_whonix-x"
rm --force "$debug_folder/usr_share_doc_whonix-x/file_list"
sudo $SUDO_OPTS touch "$debug_folder/usr_share_doc_whonix-x/file_list"
local i
i="0"
local file_name
shopt -s nullglob dotglob
for file_name in "$mount_folder/usr/share/doc/whonix-"*"/changelog.Debian.gz"; do
i="$(( $i + 1 ))"
echo "file_name number: $i | file_name: $file_name" >> "$debug_folder/usr_share_doc_whonix-x/file_list"
cp_to_user "$file_name" "$debug_folder/usr_share_doc_whonix-x/changelog.Debian.gz.$i"
done
unset file_name
shopt -u nullglob dotglob
## }}}
sudo $SUDO_OPTS mkdir --parents "$debug_folder/var_lib_dpkg_info_whonix-x"
## {{{ /var/lib/dpkg/info/whonix-*.md5sums
local file_name
shopt -s nullglob dotglob
for file_name in "$mount_folder/var/lib/dpkg/info/whonix-"*".md5sums"; do
cp_to_user "$file_name" "$debug_folder/var_lib_dpkg_info_whonix-x/"
done
unset file_name
shopt -u nullglob dotglob
## }}}
## {{{ /var/cache/debconf/*
sudo $SUDO_OPTS mkdir --parents "$debug_folder/var_cache_debconf"
local file_name
shopt -s nullglob dotglob
for file_name in "$mount_folder/var/cache/debconf/"*; do
cp_to_user "$file_name" "$debug_folder/var_cache_debconf/"
done
unset file_name
shopt -u nullglob dotglob
## }}}
## {{{ /var/lib/anon-dist/grub-backup/*
sudo $SUDO_OPTS mkdir --parents "$debug_folder/var_lib_anon-dist_grub-backup"
local file_name
shopt -s nullglob dotglob
for file_name in "$mount_folder/var/lib/anon-dist/grub-backup/"*; do
cp_to_user "$file_name" "$debug_folder/var_lib_anon-dist_grub-backup/"
done
unset file_name
shopt -u nullglob dotglob
## }}}
## "|| true", because these files are expected to be deleted.
## Copying them out of the image for easier analysis just in case.
cp_to_user "$mount_folder/var/cache/apt/pkgcache.bin" "$debug_folder/var_cache_apt_pkgcache.bin" || true
cp_to_user "$mount_folder/var/lib/dpkg/available" "$debug_folder/var_lib_dpkg_available" || true
cp_to_user "$mount_folder/var/lib/dpkg/available-old" "$debug_folder/var_lib_dpkg_available-old" || true
true "${bold}${cyan}INFO: Parsed special files. ${reset}"
}
parse_initrd() {
true "${bold}${cyan}INFO: Parsing initrd... ${reset}"
## {{{ /boot/*
## Debugging.
ls -la "$mount_folder/boot/" || true
local file_name
shopt -s nullglob dotglob
for file_name in "$mount_folder/boot/initrd.img"*; do
cp_to_user "$file_name" "$initrd_folder"
done
unset file_name
shopt -u nullglob dotglob
## }}}
## {{{ $initrd_folder/*
shopt -s nullglob dotglob
local file_name
for file_name in "$initrd_folder/"*; do
cd "$extracted_initrd_folder"
true "file_name: $file_name"
local basename_file
basename_file="$(basename "$file_name")"
sudo $SUDO_OPTS mkdir --parents "$basename_file"
cd "$basename_file"
sudo $SUDO_OPTS gzip -dc < "$file_name" | sudo $SUDO_OPTS cpio -i
done
unset file_name
shopt -u nullglob dotglob
## }}}
true "${bold}${cyan}INFO: Parsed initrd. ${reset}"
}
parse_folder() {
local i
i="0"
local sub_folder
sub_folder="$(basename "$folder/")"
local file_name
true "${bold}${cyan}INFO: Parsing $sub_folder... ${reset}"
true "${cyan}${under}This will take a while.${eunder} This is a security feature. To learn more, read:
https://www.whonix.org/wiki/Verifiable_Builds${reset}"
if [ "$disable_xtrace" = "1" ]; then
true "\
${cyan}INFO: \"set +x\", because output would be too verbose. Feel \
free to comment this out.${reset}"
true "\
${cyan}INFO: To view the progress of this or to check if it hangs, \
you could run:${reset}
ps aux | grep sha512sum
${cyan}(several times in a row) in another terminal window or watch the log using:${reset}
tail -f $report_file
"
set +x
fi
## read: using "-d ''" for NUL-delimited output.
while read -r -d '' file_name; do
i="$(( $i + 1 ))"
local absolute_file_name
absolute_file_name="${file_name#"$folder"}"
## Too verbose.
#echo "$absolute_file_name: $file_name"
echo "$absolute_file_name" >> "$file_list_file"
local skip_hash="0"
local dir_name
if [ -d "$file_name" ]; then
## $file_name is a directory.
dir_name="$file_name"
skip_hash="directory"
else
## $file_name is not a directory.
dir_name="${file_name%/*}"
fi
local dir_name_absolute
dir_name_absolute="${dir_name#"$folder"}"
## Check if $file_name exists. Somehow on Debian Wheezy it does not exit 0 for
## symlinks.
if [ ! -e "$file_name" ]; then
## Check if $file_name is a symlink.
if [ -h "$file_name" ]; then
## Symlinks are parsed below.
true
else
msg="($sub_folder) $absolute_file_name | $file_name does_not_exist"
echo "${cyan} $msg ${reset}"
echo "$msg" >> "$report_file"
if [ "$ANON_BUILD_INSTALL_TO_ROOT" = "1" ]; then
## File might no longer exist on bare metal.
## For example "sudo stat /proc/13165" could fail.
continue
else
## Should not happen inside virtual machine images, because /proc
## is not mounted there.
set -x
error "$msg"
fi
fi
fi
## Get exit code and output of "stat".
local stat_exit_code
stat_exit_code="0"
local stat_output
stat_output="$(stat \
--format "\
Access_rights_in_octal: %a \
Number_of_blocks_allocated: %b \
The_size_in_bytes_of_each_block_reported_by_b: %B \
Raw_mode_in_hex: %f \
Group_ID_of_owner: %g \
Number_of_hard_links: %h \
IO_block_size: %o \
Total_size_in_bytes: %s \
Major_device_type_in_hex: %t \
Minor_device_type_in_hex: %T \
User_ID_of_owner: %u \
Type_in_hex: %t" "$file_name" 2>&1)" \
|| { stat_exit_code="${PIPESTATUS[0]}" ; true; };
## Initialize local variables.
local \
temp1="" Access_rights_in_octal="" \
temp2="" Number_of_blocks_allocated="" \
temp3="" The_size_in_bytes_of_each_block_reported_by_b="" \
temp4="" Raw_mode_in_hex="" \
temp5="" Group_ID_of_owner="" \
temp6="" Number_of_hard_links="" \
temp7="" IO_block_size="" \
temp8="" Total_size_in_bytes="" \
temp9="" Major_device_type_in_hex="" \
temp10="" Minor_device_type_in_hex="" \
temp11="" User_ID_of_owner="" \
temp12="" Type_in_hex="" \
## Read local variables from stat_output.
read -r \
temp1 Access_rights_in_octal \
temp2 Number_of_blocks_allocated \
temp3 The_size_in_bytes_of_each_block_reported_by_b \
temp4 Raw_mode_in_hex \
temp5 Group_ID_of_owner \
temp6 Number_of_hard_links \
temp7 IO_block_size \
temp8 Total_size_in_bytes \
temp9 Major_device_type_in_hex \
temp10 Minor_device_type_in_hex \
temp11 User_ID_of_owner \
temp12 Type_in_hex \
_ \
<<< "$stat_output"
## Debugging.
#true "
#$temp1 Access_rights_in_octal
#$temp2 Number_of_blocks_allocated
#$temp3 The_size_in_bytes_of_each_block_reported_by_b
#$temp4 Raw_mode_in_hex
#$temp5 Group_ID_of_owner
#$temp6 Number_of_hard_links
#$temp7 IO_block_size
#$temp8 Total_size_in_bytes
#$temp9 Major_device_type_in_hex
#$temp10 Minor_device_type_in_hex
#$temp11 User_ID_of_owner
#$temp12 Type_in_hex
#"
## Debugging.
#true "
#$temp1 $Access_rights_in_octal
#$temp2 $Number_of_blocks_allocated
#$temp3 $The_size_in_bytes_of_each_block_reported_by_b
#$temp4 $Raw_mode_in_hex
#$temp5 $Group_ID_of_owner
#$temp6 $Number_of_hard_links
#$temp7 $IO_block_size
#$temp8 $Total_size_in_bytes
#$temp9 $Major_device_type_in_hex
#$temp10 $Minor_device_type_in_hex
#$temp11 $User_ID_of_owner
#$temp12 $Type_in_hex
#"
## $Number_of_blocks_allocated and $Total_size_in_bytes are not
## deterministic for directories. Leaving them out.
stat_output_log_directory="\
$temp1 $Access_rights_in_octal | \
$temp3 $The_size_in_bytes_of_each_block_reported_by_b | \
$temp4 $Raw_mode_in_hex | \
$temp5 $Group_ID_of_owner | \
$temp6 $Number_of_hard_links | \
$temp7 $IO_block_size | \
$temp9 $Major_device_type_in_hex | \
$temp10 $Minor_device_type_in_hex | \
$temp11 $User_ID_of_owner | \
$temp12 $Type_in_hex\
"
## $Number_of_blocks_allocated is non deterministic for some files such
## as /usr/lib/python2.7/dist-packages/twisted/web/test/test_http.pyc.
## Therefore leaving it out. Alternatively, we could remove these files
## using the cleanup chroot-post.d script and re-create them later using
## Whonix First Run Initializer.
stat_output_log_file="\
$temp1 $Access_rights_in_octal | \
$temp3 $The_size_in_bytes_of_each_block_reported_by_b | \
$temp4 $Raw_mode_in_hex | \
$temp5 $Group_ID_of_owner | \
$temp6 $Number_of_hard_links | \
$temp7 $IO_block_size | \
$temp8 $Total_size_in_bytes | \
$temp9 $Major_device_type_in_hex | \
$temp10 $Minor_device_type_in_hex | \
$temp11 $User_ID_of_owner | \
$temp12 $Type_in_hex\
"
## Check if $file_name is a symlink.
if [ -h "$file_name" ]; then
## Symlink found...
## Not using readlink with -f, so we can compare the relative links.
local file_link
local read_link_exit_code
read_link_exit_code="0"
file_link="$(readlink "$file_name" 2>&1)" || { read_link_exit_code="${PIPESTATUS[0]}" ; true; };
echo "($sub_folder) $absolute_file_name | read_link_exit_code: $read_link_exit_code | stat_output_log_file: $stat_output_log_file | stat_exit_code: $stat_exit_code | symlinks_to $file_link" >> "$report_file"
continue
fi
if [ "$skip" = "1" ]; then
local file_extension
file_extension="${file_name##*.}"
if \
[ "$file_extension" = "vmdk" ] || \
[ "$file_extension" = "mf" ] || \
[ "$file_extension" = "ovf" ] || \
[ "$file_extension" = "qcow" ] || \
[ "$file_extension" = "qcow2" ] || \
[ "$file_extension" = "vdi" ] || \
[ "$file_extension" = "raw" ] || \
[ "$file_extension" = "img" ] \
; then
## Skipping creating a sha512sum of the vmdk, because that wastes
## a lot time and we know in advance, there there will be
## differences. (Because there are no deterministically built
## operating systems yet.) We mount and analyze that image later,
## which is the whole point of this script.
##
## Skipping creation of sha512sum for mf and ovf file because those
## contain checksums and disk uuids which we expect to differ.
## Auditors are advised to manually diff those files.
##
## Still useful to have the file name and "skipped" in the report
## file, because this is a reminder to diff the mf and the ofv file
## and because the ova could also contain more than the three
## expected files.
skip_hash="skip_extension_on_demand"
fi
fi
if [ "$Number_of_blocks_allocated" = "0" ]; then
## Check if Number_of_blocks_allocated and Total_size_in_bytes are 0,
## to avoid any kind of trickery.
if [ "$Total_size_in_bytes" = "0" ]; then
## Do the probably most expensive (from performance view)
## conditions check ("/dev/"**) at the end.
if [[ "$absolute_file_name" == "/dev/"** ]]; then
## File in /dev with size of 0. No need to build a checksum of
## these. Also it would be difficult, because for example:
## "sudo sha512sum /dev/console",
## "sudo sha512sum /dev/xconsole",
## "sudo sha512sum /dev/initctl" or
## "sudo sha512sum /dev/full" would run forever.
echo "($sub_folder) $absolute_file_name | skipped_0 | stat_output_log_file: $stat_output_log_file | stat_exit_code: $stat_exit_code" >> "$report_file"
continue
fi
fi
fi
local maybe_timeout
if [ "$ANON_BUILD_INSTALL_TO_ROOT" = "1" ]; then
if [[ "$absolute_file_name" == "/dev/"** ]]; then
## For example "sudo sha512sum /dev/tty1" would run forever.
maybe_timeout="timeout --kill-after=1 1"
elif [[ "$absolute_file_name" == "/proc/"** ]]; then
## For example "sudo sha512sum /proc/kmsg" would run forever.
maybe_timeout="timeout --kill-after=1 1"
elif [[ "$absolute_file_name" == "/run/"** ]]; then
## For example "sha512sum /run/acpi_fakekey" would run forever.
maybe_timeout="timeout --kill-after=1 1"
elif [[ "$absolute_file_name" == "/tmp/"** ]]; then
## For example "sudo sha512sum /tmp/20140115122244.19545.trace" would run forever.
maybe_timeout="timeout --kill-after=1 1"
else
maybe_timeout=""
fi
else
maybe_timeout=""
fi
local checksum_output checksum_exit_code
if [ ! "$skip_hash" = "0" ]; then
checksum_exit_code="$skip_hash"
checksum_output="$skip_hash"
else
checksum_exit_code="0"
checksum_output="$($maybe_timeout sha512sum "$file_name" 2>&1)" || { checksum_exit_code="${PIPESTATUS[0]}" ; true; };
fi
if [ ! "$checksum_exit_code" = "0" ]; then
## Handling cases such as:
## - sha512sum /home/user/whonix_binary/$tempfolder/mount_folder/dev/ida/c2d10p14
## sha512sum: /home/user/whonix_binary/$tempfolder/mount_folder/dev/ida/c2d10p14: Permission denied
## - [ ! "$skip_hash" = "0" ]
if [ "$checksum_output" = "" ]; then
checksum_output="sha512sum_echoed_nothing"
fi
if [ "$skip_hash" = "directory" ]; then
echo "($sub_folder) $absolute_file_name | checksum_exit_code: $checksum_exit_code | stat_output_log_directory: $stat_output_log_directory | stat_exit_code: $stat_exit_code | checksum_output: $checksum_output" >> "$report_file"
else
echo "($sub_folder) $absolute_file_name | checksum_exit_code: $checksum_exit_code | stat_output_log_file: $stat_output_log_file | stat_exit_code: $stat_exit_code | checksum_output: $checksum_output" >> "$report_file"
fi
continue
else
local checksum
read -r checksum _ <<< "$checksum_output"
echo "($sub_folder) $absolute_file_name | checksum_exit_code: $checksum_exit_code | stat_output_log_file: $stat_output_log_file | stat_exit_code: $stat_exit_code | checksum: $checksum" >> "$report_file"
continue
fi
## The output of "find" when run on different images is non-deterministic,
## therefore piping it through "sort".
## find: using "-print0" for NUL-delimited output.
## sort: using "-z" for NUL-delimited output.
done < <(find "$folder/" -print0 | sort -z)
set -x
true "${bold}${cyan}INFO: Parsed $sub_folder. ${reset}"
}
parse_file_list_file() {
sha512sum "$file_list_file" >> "$report_file"
}
parse_endcomment() {
if [ "$endcomment" = "" ]; then
endcomment="INFO: No endcomment. | $whonix_build_error_counter error(s) detected."
fi
endcomment="\
################################################################################
$endcomment"
echo "$endcomment" >> "$report_file"
}
end() {
true "${bold}${cyan}INFO: End. | $whonix_build_error_counter error(s) detected. ${reset}"
}
main() {
parse_cmd "$@"
trap "error_handler" ERR INT TERM
preparation
parse_topcomment
extract_ova
convert_vmdk_to_vdi
convert_x_to_raw
parse_file_system
parse_vbr
parse_mbr
## sets: mount_raw_exit_code
mount_image
if [ "$mount_raw_exit_code" = "0" ]; then
parse_special_files
parse_initrd
fi
## skip, because we expect these files to be non-deterministic.
## Manual analysis required.
skip="1"
folder="$manual_analysis_folder"
parse_folder
skip="0"
if [ "$mount_raw_exit_code" = "0" ]; then
## skip, because we expect the image to be non-deterministic.
skip="1"
fi
folder="$extracted_ova_folder"
parse_folder
skip="0"
if [ "$mount_raw_exit_code" = "0" ]; then
## skip, because we expect the image to be non-deterministic.
skip="1"
fi
folder="$vdi_folder"
parse_folder
skip="0"
if [ "$mount_raw_exit_code" = "0" ]; then
## skip, because we expect the image to be non-deterministic.
skip="1"
fi
folder="$qcow_folder"
parse_folder
skip="0"
if [ "$mount_raw_exit_code" = "0" ]; then
## skip, because we expect the image to be non-deterministic.
skip="1"
fi
folder="$qcow2_folder"
parse_folder
skip="0"
if [ "$mount_raw_exit_code" = "0" ]; then
## skip, because we expect the image to be non-deterministic.
skip="1"
fi
folder="$raw_folder"
parse_folder
skip="0"
skip="0"
folder="$auto_hash_folder"
parse_folder
skip="0"
skip="0"
folder="$debug_folder"
parse_folder
skip="0"
skip="0"
folder="$initrd_folder"
parse_folder
skip="0"
skip="0"
folder="$extracted_initrd_folder"
parse_folder
skip="0"
if [ "$mount_raw_exit_code" = "0" ]; then
disable_xtrace="1"
skip="0"
folder="$mount_folder"
parse_folder
skip="0"
disable_xtrace="0"
fi
if [ "$mount_raw_exit_code" = "0" ]; then
unmount_image
fi
parse_file_list_file
parse_endcomment
end
}
main "$@"