Skip to content

Improve logging #760

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
joe-lawrence merged 8 commits into from
Nov 17, 2017
Merged

Improve logging #760

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
c33adf0
kpatch-build: Remove unnecessary 'make prepare'
juergh Nov 8, 2017
15cdcc8
kpatch-build: Add -n, --name to the usage
juergh Nov 8, 2017
862327d
kpatch-build: Fix indentation
juergh Nov 8, 2017
1950c19
kpatch-build: Add -a, --archversion commandline option
juergh Nov 8, 2017
7770c18
kpatch-build: Fix the logfile checks for 'undefined'
juergh Nov 9, 2017
b5f77d0
kpatch-build: Stop reading vmlinux after the first match
juergh Nov 9, 2017
b2c00b1
kpatch-build: Add a flexible logger function
juergh Nov 7, 2017
757bc71
kpatch-build: Add additional debug modes
juergh Nov 16, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
206 changes: 126 additions & 80 deletions kpatch-build/kpatch-build
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,10 +34,11 @@
# - Builds the patched objects with gcc flags -f[function|data]-sections
# - Runs kpatch tools to create and link the patch kernel module

set -o pipefail

BASE="$PWD"
SCRIPTDIR="$(readlink -f "$(dirname "$(type -p "$0")")")"
ARCH="$(uname -m)"
ARCHVERSION="$(uname -r)"
CPUS="$(getconf _NPROCESSORS_ONLN)"
CACHEDIR="${CACHEDIR:-$HOME/.kpatch}"
SRCDIR="$CACHEDIR/src"
Expand Down Expand Up @@ -72,12 +73,24 @@ die() {
exit 1
}

logger() {
local to_stdout=${1:-0}

if [[ $DEBUG -ge 2 ]] || [[ "$to_stdout" -eq 1 ]]; then
# Log to both stdout and the logfile
tee -a "$LOGFILE"
else
# Log only to the logfile
cat >> "$LOGFILE"
fi
}

apply_patches() {
local patch

for patch in "${PATCH_LIST[@]}"; do
patch -N -p1 --dry-run < "$patch" >> "$LOGFILE" 2>&1 || die "$patch file failed to apply"
patch -N -p1 < "$patch" >> "$LOGFILE" 2>&1 || die "$patch file failed to apply"
patch -N -p1 --dry-run < "$patch" 2>&1 | logger || die "$patch file failed to apply"
patch -N -p1 < "$patch" 2>&1 | logger || die "$patch file failed to apply"
(( APPLIED_PATCHES++ ))
done
}
Expand Down Expand Up @@ -174,6 +187,7 @@ find_core_symvers() {
gcc_version_from_file() {
readelf -p .comment "$1" | grep -o 'GCC:.*'
}

gcc_version_check() {
local c="$TEMPDIR/test.c" o="$TEMPDIR/test.o"
local out gccver kgccver
Expand Down Expand Up @@ -207,26 +221,26 @@ gcc_version_check() {

find_special_section_data_ppc64le() {
SPECIAL_VARS="$(readelf -wi "$VMLINUX" |
gawk --non-decimal-data '
BEGIN { f = b = e = 0 }
gawk --non-decimal-data '
BEGIN { f = b = e = 0 }

# Set state if name matches
f == 0 && /DW_AT_name.* fixup_entry[[:space:]]*$/ {f = 1; next}
b == 0 && /DW_AT_name.* bug_entry[[:space:]]*$/ {b = 1; next}
e == 0 && /DW_AT_name.* exception_table_entry[[:space:]]*$/ {e = 1; next}
# Set state if name matches
f == 0 && /DW_AT_name.* fixup_entry[[:space:]]*$/ {f = 1; next}
b == 0 && /DW_AT_name.* bug_entry[[:space:]]*$/ {b = 1; next}
e == 0 && /DW_AT_name.* exception_table_entry[[:space:]]*$/ {e = 1; next}

# Reset state unless this abbrev describes the struct size
f == 1 && !/DW_AT_byte_size/ { f = 0; next }
b == 1 && !/DW_AT_byte_size/ { b = 0; next }
e == 1 && !/DW_AT_byte_size/ { e = 0; next }
# Reset state unless this abbrev describes the struct size
f == 1 && !/DW_AT_byte_size/ { f = 0; next }
b == 1 && !/DW_AT_byte_size/ { b = 0; next }
e == 1 && !/DW_AT_byte_size/ { e = 0; next }

# Now that we know the size, stop parsing for it
f == 1 {printf("export FIXUP_STRUCT_SIZE=%d\n", $4); a = 2}
b == 1 {printf("export BUG_STRUCT_SIZE=%d\n", $4); b = 2}
e == 1 {printf("export EX_STRUCT_SIZE=%d\n", $4); e = 2}
# Now that we know the size, stop parsing for it
f == 1 {printf("export FIXUP_STRUCT_SIZE=%d\n", $4); a = 2}
b == 1 {printf("export BUG_STRUCT_SIZE=%d\n", $4); b = 2}
e == 1 {printf("export EX_STRUCT_SIZE=%d\n", $4); e = 2}

# Bail out once we have everything
f == 2 && b == 2 && e == 2 {exit}')"
# Bail out once we have everything
f == 2 && b == 2 && e == 2 {exit}')"

[[ -n "$SPECIAL_VARS" ]] && eval "$SPECIAL_VARS"

Expand All @@ -245,29 +259,29 @@ find_special_section_data() {

[[ "$CONFIG_PARAVIRT" -eq 0 ]] && AWK_OPTIONS="-vskip_p=1"
SPECIAL_VARS="$(readelf -wi "$VMLINUX" |
gawk --non-decimal-data $AWK_OPTIONS '
BEGIN { a = b = p = e = 0 }

# Set state if name matches
a == 0 && /DW_AT_name.* alt_instr[[:space:]]*$/ {a = 1; next}
b == 0 && /DW_AT_name.* bug_entry[[:space:]]*$/ {b = 1; next}
p == 0 && /DW_AT_name.* paravirt_patch_site[[:space:]]*$/ {p = 1; next}
e == 0 && /DW_AT_name.* exception_table_entry[[:space:]]*$/ {e = 1; next}

# Reset state unless this abbrev describes the struct size
a == 1 && !/DW_AT_byte_size/ { a = 0; next }
b == 1 && !/DW_AT_byte_size/ { b = 0; next }
p == 1 && !/DW_AT_byte_size/ { p = 0; next }
e == 1 && !/DW_AT_byte_size/ { e = 0; next }

# Now that we know the size, stop parsing for it
a == 1 {printf("export ALT_STRUCT_SIZE=%d\n", $4); a = 2}
b == 1 {printf("export BUG_STRUCT_SIZE=%d\n", $4); b = 2}
p == 1 {printf("export PARA_STRUCT_SIZE=%d\n", $4); p = 2}
e == 1 {printf("export EX_STRUCT_SIZE=%d\n", $4); e = 2}

# Bail out once we have everything
a == 2 && b == 2 && (p == 2 || skip_p) && e == 2 {exit}')"
gawk --non-decimal-data $AWK_OPTIONS '
BEGIN { a = b = p = e = 0 }

# Set state if name matches
a == 0 && /DW_AT_name.* alt_instr[[:space:]]*$/ {a = 1; next}
b == 0 && /DW_AT_name.* bug_entry[[:space:]]*$/ {b = 1; next}
p == 0 && /DW_AT_name.* paravirt_patch_site[[:space:]]*$/ {p = 1; next}
e == 0 && /DW_AT_name.* exception_table_entry[[:space:]]*$/ {e = 1; next}

# Reset state unless this abbrev describes the struct size
a == 1 && !/DW_AT_byte_size/ { a = 0; next }
b == 1 && !/DW_AT_byte_size/ { b = 0; next }
p == 1 && !/DW_AT_byte_size/ { p = 0; next }
e == 1 && !/DW_AT_byte_size/ { e = 0; next }

# Now that we know the size, stop parsing for it
a == 1 {printf("export ALT_STRUCT_SIZE=%d\n", $4); a = 2}
b == 1 {printf("export BUG_STRUCT_SIZE=%d\n", $4); b = 2}
p == 1 {printf("export PARA_STRUCT_SIZE=%d\n", $4); p = 2}
e == 1 {printf("export EX_STRUCT_SIZE=%d\n", $4); e = 2}

# Bail out once we have everything
a == 2 && b == 2 && (p == 2 || skip_p) && e == 2 {exit}')"

[[ -n "$SPECIAL_VARS" ]] && eval "$SPECIAL_VARS"

Expand Down Expand Up @@ -353,22 +367,26 @@ module_name_string() {

usage() {
echo "usage: $(basename "$0") [options] <patch1 ... patchN>" >&2
echo " patchN Input patchfile(s)" >&2
echo " -h, --help Show this help message" >&2
echo " -r, --sourcerpm Specify kernel source RPM" >&2
echo " -s, --sourcedir Specify kernel source directory" >&2
echo " -c, --config Specify kernel config file" >&2
echo " -v, --vmlinux Specify original vmlinux" >&2
echo " -j, --jobs Specify the number of make jobs" >&2
echo " -t, --target Specify custom kernel build targets" >&2
echo " -o, --output Specify output folder" >&2
echo " -d, --debug Keep scratch files in /tmp" >&2
echo " patchN Input patchfile(s)" >&2
echo " -h, --help Show this help message" >&2
echo " -a, --archversion Specify the kernel arch version" >&2
echo " -r, --sourcerpm Specify kernel source RPM" >&2
echo " -s, --sourcedir Specify kernel source directory" >&2
echo " -c, --config Specify kernel config file" >&2
echo " -v, --vmlinux Specify original vmlinux" >&2
echo " -j, --jobs Specify the number of make jobs" >&2
echo " -t, --target Specify custom kernel build targets" >&2
echo " -n, --name Specify the name of the kpatch module" >&2
echo " -o, --output Specify output folder" >&2
echo " -d, --debug Enable 'xtrace' and keep scratch files" >&2
echo " in <CACHEDIR>/tmp" >&2
echo " (can be specified multiple times)" >&2
echo " --skip-cleanup Skip post-build cleanup" >&2
echo " --skip-gcc-check Skip gcc version matching check" >&2
echo " (not recommended)" >&2
}

options="$(getopt -o hr:s:c:v:j:t:n:o:d -l "help,sourcerpm:,sourcedir:,config:,vmlinux:,jobs:,target:,name:,output:,debug,skip-gcc-check,skip-cleanup" -- "$@")" || die "getopt failed"
options="$(getopt -o ha:r:s:c:v:j:t:n:o:d -l "help,archversion:,sourcerpm:,sourcedir:,config:,vmlinux:,jobs:,target:,name:,output:,debug,skip-gcc-check,skip-cleanup" -- "$@")" || die "getopt failed"

eval set -- "$options"

Expand All @@ -378,13 +396,14 @@ while [[ $# -gt 0 ]]; do
usage
exit 0
;;
-a|--archversion)
ARCHVERSION="$2"
shift
;;
-r|--sourcerpm)
[[ ! -f "$2" ]] && die "source rpm '$2' not found"
SRCRPM="$(readlink -f "$2")"
shift
rpmname="$(basename "$SRCRPM")"
ARCHVERSION="${rpmname%.src.rpm}.$(uname -m)"
ARCHVERSION="${ARCHVERSION#kernel-}"
;;
-s|--sourcedir)
[[ ! -d "$2" ]] && die "source dir '$2' not found"
Expand Down Expand Up @@ -420,9 +439,10 @@ while [[ $# -gt 0 ]]; do
shift
;;
-d|--debug)
echo "DEBUG mode enabled"
DEBUG=1
set -o xtrace
DEBUG=$((DEBUG + 1))
if [[ $DEBUG -eq 1 ]]; then
echo "DEBUG mode enabled"
fi
;;
--skip-cleanup)
echo "Skipping cleanup"
Expand All @@ -447,21 +467,46 @@ if [[ ${#PATCH_LIST[@]} -eq 0 ]]; then
exit 1
fi

if [[ $DEBUG -eq 1 ]] || [[ $DEBUG -ge 3 ]]; then
set -o xtrace
fi

if [[ -n "$ARCHVERSION" ]] && [[ -n "$VMLINUX" ]]; then
warn "--archversion is incompatible with --vmlinux"
exit 1
fi

if [[ -n "$SRCRPM" ]]; then
if [[ -n "$ARCHVERSION" ]]; then
warn "--archversion is incompatible with --sourcerpm"
exit 1
fi
rpmname="$(basename "$SRCRPM")"
ARCHVERSION="${rpmname%.src.rpm}.$(uname -m)"
ARCHVERSION="${ARCHVERSION#kernel-}"
fi

# ensure cachedir and tempdir are setup properly and cleaned
mkdir -p "$TEMPDIR" || die "Couldn't create $TEMPDIR"
rm -rf "${TEMPDIR:?}"/*
rm -f "$LOGFILE"

if [[ -n "$USERSRCDIR" ]]; then
if [[ -n "$ARCHVERSION" ]]; then
warn "--archversion is incompatible with --sourcedir"
exit 1
fi
SRCDIR="$USERSRCDIR"

[[ -z "$VMLINUX" ]] && VMLINUX="$SRCDIR"/vmlinux
[[ ! -e "$VMLINUX" ]] && die "can't find vmlinux"

# Extract the target kernel version from vmlinux in this case.
ARCHVERSION="$(strings "$VMLINUX" | grep -e "^Linux version" | awk '{ print($3); }')"
ARCHVERSION="$(strings "$VMLINUX" | grep -m 1 -e "^Linux version" | awk '{ print($3); }')"
fi

[[ -z "$ARCHVERSION" ]] && ARCHVERSION="$(uname -r)"

[[ "$SKIPCLEANUP" -eq 0 ]] && trap cleanup EXIT INT TERM HUP

KVER="${ARCHVERSION%%-*}"
Expand All @@ -488,7 +533,7 @@ elif [[ "$DISTRO" = ubuntu ]] || [[ "$DISTRO" = debian ]]; then
if [[ "$DISTRO" = ubuntu ]]; then
[[ -e "$VMLINUX" ]] || die "linux-image-$ARCHVERSION-dbgsym not installed"

elif [[ "$DISTRO" = debian ]]; then
elif [[ "$DISTRO" = debian ]]; then
[[ -e "$VMLINUX" ]] || die "linux-image-$ARCHVERSION-dbg not installed"
fi

Expand Down Expand Up @@ -521,21 +566,21 @@ else
echo "Downloading kernel source for $ARCHVERSION"
if [[ -z "$SRCRPM" ]]; then
if [[ "$DISTRO" = fedora ]]; then
wget -P "$TEMPDIR" "http://kojipkgs.fedoraproject.org/packages/kernel/$KVER/$KREL/src/kernel-$KVER-$KREL.src.rpm" >> "$LOGFILE" 2>&1 || die
wget -P "$TEMPDIR" "http://kojipkgs.fedoraproject.org/packages/kernel/$KVER/$KREL/src/kernel-$KVER-$KREL.src.rpm" 2>&1 | logger || die
else
rpm -q --quiet yum-utils || die "yum-utils not installed"
yumdownloader --source --destdir "$TEMPDIR" "kernel-$ARCHVERSION" >> "$LOGFILE" 2>&1 || die
yumdownloader --source --destdir "$TEMPDIR" "kernel-$ARCHVERSION" 2>&1 | logger || die
fi
SRCRPM="$TEMPDIR/kernel-$KVER-$KREL.src.rpm"
fi

echo "Unpacking kernel source"

rpm -D "_topdir $RPMTOPDIR" -ivh "$SRCRPM" >> "$LOGFILE" 2>&1 || die
rpmbuild -D "_topdir $RPMTOPDIR" -bp "--target=$(uname -m)" "$RPMTOPDIR"/SPECS/kernel.spec >> "$LOGFILE" 2>&1 ||
rpm -D "_topdir $RPMTOPDIR" -ivh "$SRCRPM" 2>&1 | logger || die
rpmbuild -D "_topdir $RPMTOPDIR" -bp "--target=$(uname -m)" "$RPMTOPDIR"/SPECS/kernel.spec 2>&1 | logger ||
die "rpmbuild -bp failed. you may need to run 'yum-builddep kernel' first."

mv "$RPMTOPDIR"/BUILD/kernel-*/linux-"${ARCHVERSION%.*}"*"${ARCHVERSION##*.}" "$SRCDIR" >> "$LOGFILE" 2>&1 || die
mv "$RPMTOPDIR"/BUILD/kernel-*/linux-"${ARCHVERSION%.*}"*"${ARCHVERSION##*.}" "$SRCDIR" 2>&1 | logger || die
rm -rf "$RPMTOPDIR"
rm -rf "$SRCDIR/.git"

Expand Down Expand Up @@ -572,7 +617,7 @@ else
cd "$TEMPDIR" || die
echo "Downloading and unpacking the kernel source for $ARCHVERSION"
# Download source deb pkg
(dget -u "$url/${pkgname}/${dscname}" 2>&1) >> "$LOGFILE" || die "dget: Could not fetch/unpack $url/${pkgname}/${dscname}"
(dget -u "$url/${pkgname}/${dscname}" 2>&1) | logger || die "dget: Could not fetch/unpack $url/${pkgname}/${dscname}"
mv "${pkgname}-$KVER" "$SRCDIR" || die
cp "/boot/config-${ARCHVERSION}" "$SRCDIR/.config" || die
if [[ "$ARCHVERSION" == *-* ]]; then
Expand Down Expand Up @@ -642,14 +687,18 @@ export KCFLAGS="-I$DATADIR/patch -ffunction-sections -fdata-sections $ARCH_KCFLA
echo "Reading special section data"
find_special_section_data

if [[ $DEBUG -ge 4 ]]; then
export KPATCH_GCC_DEBUG=1
fi

echo "Building original kernel"
./scripts/setlocalversion --save-scmversion || die
make mrproper >> "$LOGFILE" 2>&1 || die
make mrproper 2>&1 | logger || die
cp -f "$CONFIGFILE" "$SRCDIR/.config"
unset KPATCH_GCC_TEMPDIR
# $TARGETS used as list, no quotes.
# shellcheck disable=SC2086
CROSS_COMPILE="$TOOLSDIR/kpatch-gcc " make "-j$CPUS" $TARGETS >> "$LOGFILE" 2>&1 || die
CROSS_COMPILE="$TOOLSDIR/kpatch-gcc " make "-j$CPUS" $TARGETS 2>&1 | logger || die

echo "Building patched kernel"
apply_patches
Expand All @@ -660,10 +709,9 @@ export KPATCH_GCC_TEMPDIR
# shellcheck disable=SC2086
CROSS_COMPILE="$TOOLSDIR/kpatch-gcc " \
KBUILD_MODPOST_WARN=1 \
make "-j$CPUS" $TARGETS >> "$LOGFILE" 2>&1 || die
[[ "${PIPESTATUS[0]}" -eq 0 ]] || die
grep -q "undefined reference" "$LOGFILE" | grep -qv kpatch_shadow && die
grep -q "undefined!" "$LOGFILE" |grep -qv kpatch_shadow && die
make "-j$CPUS" $TARGETS 2>&1 | logger || die
grep "undefined reference" "$LOGFILE" | grep -qv kpatch_shadow && die
grep "undefined!" "$LOGFILE" | grep -qv kpatch_shadow && die

if [[ ! -e "$TEMPDIR/changed_objs" ]]; then
die "no changed objects found"
Expand Down Expand Up @@ -727,7 +775,7 @@ for i in $FILES; do
if [[ -e "orig/$i" ]]; then
# create-diff-object orig.o patched.o kernel-object output.o Module.symvers patch-mod-name
"$TOOLSDIR"/create-diff-object "orig/$i" "patched/$i" "$KOBJFILE" \
"output/$i" "$SRCDIR/Module.symvers" "${MODNAME//-/_}" 2>&1 | tee -a "$LOGFILE"
"output/$i" "$SRCDIR/Module.symvers" "${MODNAME//-/_}" 2>&1 | logger 1
check_pipe_status create-diff-object
# create-diff-object returns 3 if no functional change is found
[[ "$rc" -eq 0 ]] || [[ "$rc" -eq 3 ]] || ERROR="$((ERROR + 1))"
Expand Down Expand Up @@ -763,8 +811,6 @@ if "$KPATCH_MODULE"; then
fi

echo "Building patch module: $MODNAME.ko"
cd "$SRCDIR" || die
make prepare >> "$LOGFILE" 2>&1 || die

if [[ ! -z "$UBUNTU_KERNEL" ]]; then
# UBUNTU: add UTS_UBUNTU_RELEASE_ABI to utsrelease.h after regenerating it
Expand All @@ -776,14 +822,14 @@ fi
cd "$TEMPDIR/output" || die
# $KPATCH_LDFLAGS and result of find used as list, no quotes.
# shellcheck disable=SC2086,SC2046
ld -r $KPATCH_LDFLAGS -o ../patch/tmp_output.o $(find . -name "*.o") >> "$LOGFILE" 2>&1 || die
ld -r $KPATCH_LDFLAGS -o ../patch/tmp_output.o $(find . -name "*.o") 2>&1 | logger || die

if "$KPATCH_MODULE"; then
# Add .kpatch.checksum for kpatch script
md5sum ../patch/tmp_output.o | awk '{printf "%s\0", $1}' > checksum.tmp || die
objcopy --add-section .kpatch.checksum=checksum.tmp --set-section-flags .kpatch.checksum=alloc,load,contents,readonly ../patch/tmp_output.o || die
rm -f checksum.tmp
"$TOOLSDIR"/create-kpatch-module "$TEMPDIR"/patch/tmp_output.o "$TEMPDIR"/patch/output.o 2>&1 | tee -a "$LOGFILE"
"$TOOLSDIR"/create-kpatch-module "$TEMPDIR"/patch/tmp_output.o "$TEMPDIR"/patch/output.o 2>&1 | logger 1
check_pipe_status create-kpatch-module
else
cp "$TEMPDIR"/patch/tmp_output.o "$TEMPDIR"/patch/output.o || die
Expand All @@ -794,14 +840,14 @@ cd "$TEMPDIR/patch" || die
KPATCH_BUILD="$SRCDIR" KPATCH_NAME="$MODNAME" \
KBUILD_EXTRA_SYMBOLS="$KBUILD_EXTRA_SYMBOLS" \
KPATCH_LDFLAGS="$KPATCH_LDFLAGS" \
make >> "$LOGFILE" 2>&1 || die
make 2>&1 | logger || die

if ! "$KPATCH_MODULE"; then
if [[ -z "$KPATCH_LDFLAGS" ]]; then
extra_flags="--no-klp-arch-sections"
fi
cp "$TEMPDIR/patch/$MODNAME.ko" "$TEMPDIR/patch/tmp.ko" || die
"$TOOLSDIR"/create-klp-module $extra_flags "$TEMPDIR/patch/tmp.ko" "$TEMPDIR/patch/$MODNAME.ko" 2>&1 | tee -a "$LOGFILE"
"$TOOLSDIR"/create-klp-module $extra_flags "$TEMPDIR/patch/tmp.ko" "$TEMPDIR/patch/$MODNAME.ko" 2>&1 | logger 1
check_pipe_status create-klp-module
fi

Expand Down
Loading