-
Notifications
You must be signed in to change notification settings - Fork 158
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
tests/kola: add extended upgrade test
This test will attempt to test an upgrade from a given starting point (assumed by the caller passing in a specific `cosa kola run --build=x.y.z`) all the way to the latest build that is staged to be released. The test is basic in that it essentially tests 1) updates work 2) boot works. An example invocation for this test would look like: ``` cosa buildfetch --stream=next --build=34.20210904.1.0 --artifact=qemu cosa decompress --build=34.20210904.1.0 cosa kola run --build=34.20210904.1.0 --tag extended-upgrade ``` You can then monitor the console.txt or journal.txt to watch the upgrades happen via `tail -f tmp/kola/ext.config.upgrade.extened/*/console.txt'. Another useful thing to do is to just monitor the major events: ``` $ tail -f tmp/kola/ext.config.upgrade.extended/*/journal.txt | grep --color -i 'ok reached version' Mar 20 02:25:12.584096 kola-runext-test.sh[2085]: ok Reached version: 34.20210904.1.0 Mar 20 02:26:37.107738 kola-runext-test.sh[3538]: ok Reached version: 35.20210924.1.0 Mar 20 02:28:08.509482 kola-runext-test.sh[1668]: ok Reached version: 36.20220418.1.0 Mar 20 02:28:41.242555 kola-runext-test.sh[1649]: ok Reached version: 36.20220906.1.0 Mar 20 02:29:33.635979 kola-runext-test.sh[1635]: ok Reached version: 37.20221111.1.0 Mar 20 02:30:16.657854 kola-runext-test.sh[1469]: ok Reached version: 37.20230303.1.0 ```
- Loading branch information
Showing
3 changed files
with
183 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,24 @@ | ||
variant: fcos | ||
# Must use version 1.0.0 here to use Ignition spec 3.0.0 for | ||
# our oldest supported starting points (i.e. 31.20200108.3.0) | ||
version: 1.0.0 | ||
storage: | ||
files: | ||
- path: /etc/systemd/journald.conf.d/forward-to-console.conf | ||
mode: 0644 | ||
contents: | ||
inline: | | ||
# Send journal messages to the console. This is so we can see the | ||
# progress even after autopkgtest-reboot-prepare has been called. | ||
[Journal] | ||
ForwardToConsole=yes | ||
- path: /etc/zincati/config.d/99-config.toml | ||
mode: 0644 | ||
contents: | ||
inline: | | ||
# Don't wait for any rollout window, update ASAP | ||
[identity] | ||
rollout_wariness = 0.0000 | ||
# Increase the frequency at which we check for updates | ||
[agent.timing] | ||
steady_interval_secs = 20 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1 @@ | ||
../../../data/commonlib.sh |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,158 @@ | ||
#!/bin/bash | ||
## kola: | ||
## # - needs-internet: to pull updates | ||
## tags: "needs-internet" | ||
## # Extend the timeout since a lot of updates/reboots can happen. | ||
## timeoutMin: 45 | ||
## # Only run this test when specifically requested. | ||
## requiredTag: extended-upgrade | ||
|
||
set -eux -o pipefail | ||
|
||
. $KOLA_EXT_DATA/commonlib.sh | ||
|
||
# This test will attempt to test an upgrade from a given starting | ||
# point (assumed by the caller passing in a specific | ||
# `cosa kola run --build=x.y.z`) all the way to the latest build | ||
# that is staged to be released. The test is basic in that it | ||
# essentially tests 1) updates work 2) boot works. | ||
# | ||
# An example invocation for this test would look like: | ||
|
||
# ``` | ||
# cosa buildfetch --stream=next --build=34.20210904.1.0 --artifact=qemu | ||
# cosa decompress --build=34.20210904.1.0 | ||
# cosa kola run --build=34.20210904.1.0 --tag extended-upgrade | ||
# ``` | ||
# | ||
# You can monitor the progress from the console and journal: | ||
# - everything: | ||
# - tail -f tmp/kola/ext.config.upgrade.extended/*/console.txt | ||
# - major events: | ||
# - tail -f tmp/kola/ext.config.upgrade.extended/*/journal.txt | grep --color -i 'ok reached version' | ||
# | ||
# For convenience, here is a list of the earliest releases on each | ||
# stream/architecture: | ||
# | ||
# stable | ||
# - x86_64 31.20200108.3.0 | ||
# - aarch64 34.20210821.3.0 | ||
# - s390x 36.20220618.3.1 | ||
# testing | ||
# - x86_64 30.20190716.1 | ||
# - aarch64 34.20210904.2.0 | ||
# - s390x 36.20220618.2.0 | ||
# next | ||
# - x86_64 32.20200416.1.0 | ||
# - aarch64 34.20210904.1.0 | ||
# - s390x 36.20220618.1.1 | ||
|
||
. /etc/os-release # for $VERSION_ID | ||
|
||
# delete the disabling of updates that was done by the test framework | ||
if [ -f /etc/zincati/config.d/90-disable-auto-updates.toml ]; then | ||
rm -f /etc/zincati/config.d/90-disable-auto-updates.toml | ||
systemctl restart zincati | ||
fi | ||
|
||
version=$(rpm-ostree status --json | jq -r '.deployments[0].version') | ||
stream=$(rpm-ostree status --json | jq -r '.deployments[0]["base-commit-meta"]["fedora-coreos.stream"]') | ||
|
||
test -f /srv/releases.json || \ | ||
curl -L "https://builds.coreos.fedoraproject.org/prod/streams/${stream}/releases.json" > /srv/releases.json | ||
test -f /srv/builds.json || \ | ||
curl -L "https://builds.coreos.fedoraproject.org/prod/streams/${stream}/builds/builds.json" > /srv/builds.json | ||
|
||
last_release=$(jq -r .releases[-1].version /srv/releases.json) | ||
target_version=$(jq -r .builds[0].id /srv/builds.json) | ||
|
||
grab-gpg-keys() { | ||
# For older FCOS we had an issue where when we tried to pull the | ||
# commits from the repo it would fail if we were on N-2 because | ||
# the newer commits would be signed with a key the old OS didn't | ||
# know anything about. We applied a workaround in newer releases, | ||
# so this workaround should be limited to zincati older than v0.0.24 | ||
# https://github.com/coreos/fedora-coreos-tracker/issues/749 | ||
max_version=${target_version:0:2} # i.e. 36, 37, 38, etc.. | ||
for ver in $(seq $VERSION_ID $max_version); do | ||
test -e "/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-${ver}-primary" && continue | ||
curl -L "https://src.fedoraproject.org/rpms/fedora-repos/raw/rawhide/f/RPM-GPG-KEY-fedora-${ver}-primary" | \ | ||
sudo tee "/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-${ver}-primary" | ||
sudo chcon -v --reference="/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-${VERSION_ID}-primary" "/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-${ver}-primary" | ||
done | ||
} | ||
|
||
fix-update-url() { | ||
# We switched to non stg URL in zincati v0.0.10 [1]. For older clients | ||
# we need to update the runtime configuration of zincati to get past the problem. | ||
# [1] https://github.com/coreos/zincati/commit/1d73801ccd015cdce89f082cb1eeb9b4b8335760 | ||
cat <<'EOF' > /run/zincati/config.d/50-fedora-coreos-cincinnati.toml | ||
[cincinnati] | ||
base_url= "https://updates.coreos.fedoraproject.org" | ||
EOF | ||
systemctl restart zincati | ||
} | ||
|
||
ok "Reached version: $version" | ||
|
||
# Are we all the way at the desired target version? | ||
# If so then we can exit with success! | ||
if vereq $version $target_version; then | ||
ok "Fully upgraded to $target_version" | ||
bootupctl status | ||
exit 0 | ||
fi | ||
|
||
# Apply workarounds based on the current version of the system. | ||
# | ||
# First release on each stream with new enough zincati for updates stg.fedoraprojec.org | ||
# - 31.20200505.3.0 | ||
# - 31.20200505.2.0 | ||
# - 32.20200505.1.0 | ||
# | ||
# First release with new enough zincati with workaround for N-2 gpg key issue | ||
# - 35.20211119.3.0 | ||
# - 35.20211119.2.0 | ||
# - 35.20211119.1.0 | ||
# | ||
case "$stream" in | ||
'next') | ||
verlt $version '35.20211119.1.0' && grab-gpg-keys | ||
verlt $version '31.20200505.1.0' && fix-update-url | ||
;; | ||
'testing') | ||
verlt $version '35.20211119.2.0' && grab-gpg-keys | ||
verlt $version '31.20200505.2.0' && fix-update-url | ||
;; | ||
'stable') | ||
verlt $version '35.20211119.3.0' && grab-gpg-keys | ||
verlt $version '31.20200505.3.0' && fix-update-url | ||
;; | ||
*) fatal "unexpected stream: $stream";; | ||
esac | ||
|
||
# If we have made it all the way to the last release then | ||
# we have one more test. We'll now rebase to the target | ||
# version, which should be in the compose OSTree repo. | ||
if vereq $version $last_release; then | ||
systemctl stop zincati | ||
rpm-ostree rebase fedora-compose: $target_version | ||
/tmp/autopkgtest-reboot reboot # execute the reboot | ||
sleep infinity | ||
fi | ||
|
||
# Watch the Zincati logs to see if it got a lead on a new update. | ||
# Timeout after some time if no update. Unset pipefail since the | ||
# journalctl -f will give a bad exit code when grep exits early. | ||
set +o pipefail | ||
cmd="journalctl -b 0 -f --no-tail -u zincati.service" | ||
if ! timeout 90s $cmd | grep --max-count=1 'proceeding to stage it'; then | ||
# No update initiated within timeout; let's error. | ||
fatal "Updating the system stalled out on version: $version" | ||
fi | ||
set -o pipefail | ||
|
||
|
||
# OK update has been initiated, prepare for reboot and sleep | ||
/tmp/autopkgtest-reboot-prepare reboot | ||
sleep infinity |