-
-
Notifications
You must be signed in to change notification settings - Fork 2.1k
/
prepare-host.sh
399 lines (333 loc) · 17.4 KB
/
prepare-host.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
#!/usr/bin/env bash
#
# SPDX-License-Identifier: GPL-2.0
#
# Copyright (c) 2013-2023 Igor Pecovnik, igor@armbian.com
#
# This file is a part of the Armbian Build Framework
# https://github.com/armbian/build/
# prepare_host
#
# * checks and installs necessary packages
# * creates directory structure
# * changes system settings
#
function prepare_host() {
LOG_SECTION="prepare_host_noninteractive" do_with_logging prepare_host_noninteractive
return 0
}
function assert_prepared_host() {
if [[ "${PRE_PREPARED_HOST:-"no"}" == "yes" ]]; then
return 0
fi
if [[ ${prepare_host_has_already_run:-0} -lt 1 ]]; then
exit_with_error "assert_prepared_host: Host has not yet been prepared. This is a bug in armbian-next code. Please report!"
fi
}
function check_basic_host() {
display_alert "Checking" "basic host setup" "info"
obtain_and_check_host_release_and_arch # sets HOSTRELEASE and validates it for sanity; also HOSTARCH
check_host_has_enough_disk_space # Checks disk space and exits if not enough
check_windows_wsl2 # checks if on Windows, on WSL2, (not 1) and exits if not supported
wait_for_package_manager # wait until dpkg is not locked...
}
function prepare_host_noninteractive() {
display_alert "Preparing" "host" "info"
# The 'offline' variable must always be set to 'true' or 'false'
declare offline=false
if [ "$OFFLINE_WORK" == "yes" ]; then
offline=true
fi
# fix for Locales settings, if locale-gen is installed, and /etc/locale.gen exists.
if [[ -n "$(command -v locale-gen)" && -f /etc/locale.gen ]]; then
if ! grep -q "^en_US.UTF-8 UTF-8" /etc/locale.gen; then
# @TODO: rpardini: this is bull, we're always root here. we've been pre-sudo'd.
local sudo_prefix="" && is_root_or_sudo_prefix sudo_prefix # nameref; "sudo_prefix" will be 'sudo' or ''
${sudo_prefix} sed -i 's/# en_US.UTF-8/en_US.UTF-8/' /etc/locale.gen
${sudo_prefix} locale-gen
fi
else
display_alert "locale-gen is not installed @host" "skipping locale-gen -- problems might arise" "warn"
fi
# Let's try and get all log output in English, overriding the builder's chosen or default language
export LANG="en_US.UTF-8"
export LANGUAGE="en_US.UTF-8"
export LC_ALL="en_US.UTF-8"
export LC_MESSAGES="en_US.UTF-8"
declare -g USE_LOCAL_APT_DEB_CACHE=${USE_LOCAL_APT_DEB_CACHE:-yes} # Use SRC/cache/aptcache as local apt cache by default
display_alert "Using local apt cache?" "USE_LOCAL_APT_DEB_CACHE: ${USE_LOCAL_APT_DEB_CACHE}" "debug"
# if USE_LOCAL_APT_DEB_CACHE equals no, display_alert a warning, it's not a good idea.
if [[ "${USE_LOCAL_APT_DEB_CACHE}" == "no" ]]; then
display_alert "USE_LOCAL_APT_DEB_CACHE is set to 'no'" "not recommended" "wrn"
fi
if armbian_is_running_in_container; then
display_alert "Running in container" "Adding provisions for container building" "info"
declare -g CONTAINER_COMPAT=yes # this controls mknod usage for loop devices.
if [[ "${MANAGE_ACNG}" == "yes" ]]; then
display_alert "Running in container" "Disabling ACNG - MANAGE_ACNG=yes not supported in containers" "warn"
declare -g MANAGE_ACNG=no
fi
SYNC_CLOCK=no
else
display_alert "NOT running in container" "No special provisions for container building" "debug"
fi
# If offline, do not try to install dependencies, manage acng, or sync the clock.
if ! $offline; then
# Prepare the list of host dependencies; it requires the target arch, the host release and arch
late_prepare_host_dependencies
install_host_dependencies "late dependencies during prepare_release"
# Manage apt-cacher-ng, if such is the case
[[ "${MANAGE_ACNG}" == "yes" ]] && acng_configure_and_restart_acng
# sync clock
if [[ $SYNC_CLOCK != no && -f /var/run/ntpd.pid ]]; then
display_alert "ntpd is running, skipping" "SYNC_CLOCK=no" "debug"
SYNC_CLOCK=no
fi
if [[ $SYNC_CLOCK != no ]]; then
display_alert "Syncing clock" "host" "info"
run_host_command_logged ntpdate "${NTP_SERVER:-pool.ntp.org}" || true # allow failures
fi
fi
# create directory structure # @TODO: this should be close to DEST, otherwise super-confusing
mkdir -p "${SRC}"/{cache,output} "${USERPATCHES_PATH}"
# @TODO: original: mkdir -p "${DEST}"/debs-beta/extra "${DEST}"/debs/extra "${DEST}"/{config,debug,patch} "${USERPATCHES_PATH}"/overlay "${SRC}"/cache/{sources,hash,hash-beta,toolchain,utility,rootfs} "${SRC}"/.tmp
mkdir -p "${USERPATCHES_PATH}"/overlay "${SRC}"/cache/{sources,rootfs} "${SRC}"/.tmp
# If offline, do not try to download/install toolchains.
if ! $offline; then
download_external_toolchains # Mostly deprecated, since SKIP_EXTERNAL_TOOLCHAINS=yes is the default
fi
# NEEDS_BINFMT=yes is set by default build and rootfs artifact build.
# if we're building an image, not only packages/artifacts...
# ... and the host arch does not match the target arch ...
# ... we then require binfmt_misc to be enabled.
# "enable arm binary format so that the cross-architecture chroot environment will work"
if [[ "${NEEDS_BINFMT:-"no"}" == "yes" ]]; then
if [[ "${SHOW_DEBUG}" == "yes" ]]; then
display_alert "Debugging binfmt - early" "/proc/sys/fs/binfmt_misc/" "debug"
run_host_command_logged ls -la /proc/sys/fs/binfmt_misc/ || true
fi
if dpkg-architecture -e "${ARCH}"; then
display_alert "Native arch build" "target ${ARCH} on host $(dpkg --print-architecture)" "cachehit"
else
local failed_binfmt_modprobe=0
display_alert "Cross arch build" "target ${ARCH} on host $(dpkg --print-architecture)" "debug"
# Check if binfmt_misc is loaded; if not, try to load it, but don't fail: it might be built in.
if grep -q "^binfmt_misc" /proc/modules; then
display_alert "binfmt_misc is already loaded" "binfmt_misc already loaded" "debug"
else
display_alert "binfmt_misc is not loaded" "trying to load binfmt_misc" "debug"
# try to modprobe. if it fails, emit a warning later, but not here.
# this is for the in-container case, where the host already has the module, but won't let the container know about it.
modprobe -q binfmt_misc || failed_binfmt_modprobe=1
fi
# Now, /proc/sys/fs/binfmt_misc/ has to be mounted. Mount, or fail with a message
if mountpoint -q /proc/sys/fs/binfmt_misc/; then
display_alert "binfmt_misc is already mounted" "binfmt_misc already mounted" "debug"
else
display_alert "binfmt_misc is not mounted" "trying to mount binfmt_misc" "debug"
mount -t binfmt_misc binfmt_misc /proc/sys/fs/binfmt_misc/ || {
if [[ $failed_binfmt_modprobe == 1 ]]; then
display_alert "Failed to load binfmt_misc module" "modprobe binfmt_misc failed" "wrn"
fi
display_alert "Check your HOST kernel" "CONFIG_BINFMT_MISC=m is required in host kernel" "warn"
display_alert "Failed to mount" "binfmt_misc /proc/sys/fs/binfmt_misc/" "err"
exit_with_error "Failed to mount binfmt_misc"
}
display_alert "binfmt_misc mounted" "binfmt_misc mounted" "debug"
fi
declare host_arch
host_arch="$(arch)"
local -a wanted_arches=("arm" "aarch64" "x86_64" "riscv64")
display_alert "Preparing binfmts for arch" "binfmts: host '${host_arch}', wanted arches '${wanted_arches[*]}'" "debug"
declare wanted_arch
for wanted_arch in "${wanted_arches[@]}"; do
if [[ "${host_arch}" != "${wanted_arch}" ]]; then
if [[ ! -e "/proc/sys/fs/binfmt_misc/qemu-${wanted_arch}" ]]; then
display_alert "Updating binfmts" "update-binfmts --enable qemu-${wanted_arch}" "debug"
if [[ "${host_arch}" == "aarch64" && "${wanted_arch}" == "arm" ]]; then
display_alert "Trying to update binfmts - aarch64 (sometimes) does 32-bit sans emulation" "update-binfmts --enable qemu-${wanted_arch}" "debug"
run_host_command_logged update-binfmts --enable "qemu-${wanted_arch}" "&>" "/dev/null" "||" "true" # don't fail nor produce output, which can be misleading.
else
run_host_command_logged update-binfmts --enable "qemu-${wanted_arch}" || display_alert "Failed to update binfmts" "update-binfmts --enable qemu-${wanted_arch}" "err" # log & continue on failure
fi
fi
fi
done
# @TODO: we could create a tiny loop here to test if the binfmt_misc is working, but this is before deps are installed.
fi
if [[ "${SHOW_DEBUG}" == "yes" ]]; then
display_alert "Debugging binfmt - late" "/proc/sys/fs/binfmt_misc/" "debug"
run_host_command_logged ls -la /proc/sys/fs/binfmt_misc/ || true
fi
fi
# @TODO: rpardini: this does not belong here, instead with the other templates, pre-configuration.
[[ ! -f "${USERPATCHES_PATH}"/customize-image.sh ]] && run_host_command_logged cp -pv "${SRC}"/config/templates/customize-image.sh.template "${USERPATCHES_PATH}"/customize-image.sh
if [[ -d "${USERPATCHES_PATH}" ]]; then
# create patches directory structure under USERPATCHES_PATH
find "${SRC}"/patch -maxdepth 2 -type d ! -name . | sed "s%/.*patch%/$USERPATCHES_PATH%" | xargs mkdir -p
fi
# Reset owner of userpatches if so required
reset_uid_owner "${USERPATCHES_PATH}" # Fix owner of files in the final destination
declare -i -g -r prepare_host_has_already_run=1 # global, readonly.
return 0
}
# Early: we've possibly no idea what the host release or arch we're building on, or what the target arch is. All-deps.
# Early: we've a best-guess indication of the host release, but not target. (eg: Dockerfile generate)
# Early: we're certain about the host release and arch, but not anything about the target (eg: docker build of the Dockerfile, cli-requirements)
# Late: we know everything; produce a list that is optimized for the host+target we're building. (eg: Oleg)
function early_prepare_host_dependencies() {
if [[ "x${host_release:-}x" == "xx" ]]; then
display_alert "Host release unknown" "host_release not set on call to early_prepare_host_dependencies" "warn"
fi
if [[ "x${host_arch:-}x" == "xx" ]]; then
display_alert "Host arch unknown" "host_arch not set on call to early_prepare_host_dependencies" "debug"
fi
adaptative_prepare_host_dependencies
}
function late_prepare_host_dependencies() {
[[ -z "${ARCH}" ]] && exit_with_error "ARCH is not set"
[[ -z "${HOSTRELEASE}" ]] && exit_with_error "HOSTRELEASE is not set"
[[ -z "${HOSTARCH}" ]] && exit_with_error "HOSTARCH is not set"
[[ -z "${RELEASE}" ]] && display_alert "RELEASE is not set" "defaulting to host's '${HOSTRELEASE}'" "debug"
target_arch="${ARCH}" host_release="${HOSTRELEASE}" \
host_arch="${HOSTARCH}" target_release="${RELEASE:-"${HOSTRELEASE}"}" \
early_prepare_host_dependencies
}
# Adaptive: used by both early & late.
function adaptative_prepare_host_dependencies() {
if [[ "x${host_release:-"unknown"}x" == "xx" ]]; then
display_alert "No specified host_release" "preparing for all-hosts, all-targets deps" "debug"
else
display_alert "Using passed-in host_release" "${host_release}" "debug"
fi
if [[ "x${target_arch:-"unknown"}x" == "xx" ]]; then
display_alert "No specified target_arch" "preparing for all-hosts, all-targets deps" "debug"
else
display_alert "Using passed-in target_arch" "${target_arch}" "debug"
fi
#### Common: for all releases, all host arches, and all target arches.
declare -a -g host_dependencies=(
# big bag of stuff from before
bc binfmt-support
bison
libc6-dev make dpkg-dev gcc # build-essential, without g++
ca-certificates ccache cpio
debootstrap device-tree-compiler dialog dirmngr dosfstools
dwarves # dwarves has been replaced by "pahole" and is now a transitional package
flex
gawk gnupg gpg
imagemagick # required for boot_logo, plymouth: converting images / spinners
jq # required for parsing JSON, specially rootfs-caching related.
kmod # this causes initramfs rebuild, but is usually pre-installed, so no harm done unless it's an upgrade
libbison-dev libelf-dev libfdt-dev libfile-fcntllock-perl libmpc-dev libfl-dev liblz4-tool
libncurses-dev libssl-dev libusb-1.0-0-dev
linux-base locales lsof
ncurses-base ncurses-term # for `make menuconfig`
ntpdate
patchutils pkg-config pv
qemu-user-static
rsync
swig # swig is needed for some u-boot's. example: "bananapi.conf"
u-boot-tools
udev # causes initramfs rebuild, but is usually pre-installed.
uuid-dev
zlib1g-dev
# by-category below
file tree expect # logging utilities; expect is needed for 'unbuffer' command
colorized-logs # for ansi2html, ansi2txt, pipetty
unzip zip pigz xz-utils pbzip2 lzop zstd # compressors et al
parted gdisk fdisk # partition tools @TODO why so many?
aria2 curl wget axel # downloaders et al
parallel # do things in parallel (used for fast md5 hashing in initrd cache)
)
# @TODO: distcc -- handle in extension?
### Python
host_deps_add_extra_python # See python-tools.sh::host_deps_add_extra_python()
# Python3 -- required for Armbian's Python tooling, and also for more recent u-boot builds. Needs 3.9+
host_dependencies+=("python3-dev" "python3-distutils" "python3-setuptools" "python3-pip")
# Python2 -- required for some older u-boot builds
# Debian 'sid'/'bookworm' and Ubuntu 'lunar' does not carry python2 anymore; in this case some u-boot's might fail to build.
if [[ "sid bookworm trixie lunar" == *"${host_release}"* ]]; then
display_alert "Python2 not available on host release '${host_release}'" "old(er) u-boot builds might/will fail" "wrn"
else
host_dependencies+=("python2" "python2-dev")
fi
# Only install acng if asked to.
if [[ "${MANAGE_ACNG}" == "yes" ]]; then
host_dependencies+=("apt-cacher-ng")
fi
### ARCH
declare wanted_arch="${target_arch:-"all"}"
if [[ "${wanted_arch}" == "amd64" || "${wanted_arch}" == "all" ]]; then
host_dependencies+=("gcc-x86-64-linux-gnu") # from crossbuild-essential-amd64
fi
if [[ "${wanted_arch}" == "arm64" || "${wanted_arch}" == "all" ]]; then
host_dependencies+=("gcc-aarch64-linux-gnu") # from crossbuild-essential-arm64
fi
if [[ "${wanted_arch}" == "armhf" || "${wanted_arch}" == "all" ]]; then
host_dependencies+=("gcc-arm-linux-gnueabihf" "gcc-arm-linux-gnueabi") # from crossbuild-essential-armhf crossbuild-essential-armel
fi
if [[ "${wanted_arch}" == "riscv64" || "${wanted_arch}" == "all" ]]; then
host_dependencies+=("gcc-riscv64-linux-gnu") # crossbuild-essential-riscv64 is not even available "yet"
host_dependencies+=("debian-archive-keyring")
fi
if [[ "${wanted_arch}" != "amd64" ]]; then
host_dependencies+=(libc6-amd64-cross) # Support for running x86 binaries (under qemu on other arches)
fi
declare -g EXTRA_BUILD_DEPS=""
call_extension_method "add_host_dependencies" <<- 'ADD_HOST_DEPENDENCIES'
*run before installing host dependencies*
you can add packages to install, space separated, to ${EXTRA_BUILD_DEPS} here.
ADD_HOST_DEPENDENCIES
if [ -n "${EXTRA_BUILD_DEPS}" ]; then
# shellcheck disable=SC2206 # I wanna expand. @TODO: later will convert to proper array
host_dependencies+=(${EXTRA_BUILD_DEPS})
fi
declare -g FINAL_HOST_DEPS="${host_dependencies[*]}"
call_extension_method "host_dependencies_known" <<- 'HOST_DEPENDENCIES_KNOWN'
*run after all host dependencies are known (but not installed)*
At this point we can read `${FINAL_HOST_DEPS}`, but changing won't have any effect.
All the dependencies, including the default/core deps and the ones added via `${EXTRA_BUILD_DEPS}`
are determined at this point, but not yet installed.
HOST_DEPENDENCIES_KNOWN
}
function install_host_dependencies() {
display_alert "Installing build dependencies" "$*" "debug"
# don't prompt for apt cacher selection. this is to skip the prompt only, since we'll manage acng config later.
local sudo_prefix="" && is_root_or_sudo_prefix sudo_prefix # nameref; "sudo_prefix" will be 'sudo' or ''
${sudo_prefix} echo "apt-cacher-ng apt-cacher-ng/tunnelenable boolean false" | ${sudo_prefix} debconf-set-selections
# This handles the wanted list in $host_dependencies, updates apt only if needed
# $host_dependencies is produced by early_prepare_host_dependencies()
install_host_side_packages "${host_dependencies[@]}"
run_host_command_logged update-ccache-symlinks
declare -g FINAL_HOST_DEPS="${host_dependencies[*]}"
call_extension_method "host_dependencies_ready" <<- 'HOST_DEPENDENCIES_READY'
*run after all host dependencies are installed*
At this point we can read `${FINAL_HOST_DEPS}`, but changing won't have any effect.
All the dependencies, including the default/core deps and the ones added via `${EXTRA_BUILD_DEPS}`
are installed at this point. The system clock has not yet been synced.
HOST_DEPENDENCIES_READY
unset FINAL_HOST_DEPS # don't leak this after the hook is done
}
function check_host_has_enough_disk_space() {
declare -a dirs_to_check=("${DEST}" "${SRC}/cache")
for dir in "${dirs_to_check[@]}"; do
if [[ ! -d "${dir}" ]]; then
display_alert "Directory not found" "Skipping disk space check for '${dir}'" "debug"
continue
fi
check_dir_has_enough_disk_space "${dir}" 10 || exit_if_countdown_not_aborted 10 "Low free disk space left in '${dir}'"
done
}
function check_dir_has_enough_disk_space() {
declare target="${1}"
declare -i min_free_space_gib="${2:-10}"
declare -i free_space_bytes
free_space_bytes=$(findmnt --noheadings --output AVAIL --bytes --target "${target}" --uniq 2> /dev/null) # in bytes
if [[ -n "$free_space_bytes" && $((free_space_bytes / 1073741824)) -lt $min_free_space_gib ]]; then
display_alert "Low free space left" "${target}: $((free_space_bytes / 1073741824))GiB free, ${min_free_space_gib} GiB required" "wrn"
return 1
fi
display_alert "Free space left" "${target}: $((free_space_bytes / 1073741824))GiB" "debug"
return 0
}