tests/kola: add extended upgrade test

This test will attempt to test an upgrade from a given starting
point (assumed by the caller passing in a specific
`cosa kola run --build=x.y.z`) all the way to the latest build
that is staged to be released. The test is basic in that it
essentially tests 1) updates work 2) boot works.

An example invocation for this test would look like:

```
cosa buildfetch --stream=next --build=34.20210904.1.0 --artifact=qemu
cosa decompress --build=34.20210904.1.0
cosa kola run --build=34.20210904.1.0 --tag extended-upgrade
```

You can then monitor the console.txt or journal.txt to watch the
upgrades happen via `tail -f tmp/kola/ext.config.upgrade.extened/*/console.txt'.

Another useful thing to do is to just monitor the major events:

```
$ tail -f  tmp/kola/ext.config.upgrade.extended/*/journal.txt | grep --color -i 'ok reached version'
Mar 20 02:25:12.584096 kola-runext-test.sh[2085]: ok Reached version: 34.20210904.1.0
Mar 20 02:26:37.107738 kola-runext-test.sh[3538]: ok Reached version: 35.20210924.1.0
Mar 20 02:28:08.509482 kola-runext-test.sh[1668]: ok Reached version: 36.20220418.1.0
Mar 20 02:28:41.242555 kola-runext-test.sh[1649]: ok Reached version: 36.20220906.1.0
Mar 20 02:29:33.635979 kola-runext-test.sh[1635]: ok Reached version: 37.20221111.1.0
Mar 20 02:30:16.657854 kola-runext-test.sh[1469]: ok Reached version: 37.20230303.1.0
```

tests/kola/upgrade/extended/config.bu

@@ -0,0 +1,24 @@
+variant: fcos
+# Must use version 1.0.0 here to use Ignition spec 3.0.0 for
+# our oldest supported starting points (i.e. 31.20200108.3.0)
+version: 1.0.0
+storage:
+  files:
+    - path: /etc/systemd/journald.conf.d/forward-to-console.conf
+      mode: 0644
+      contents:
+        inline: |
+          # Send journal messages to the console. This is so we can see the
+          # progress even after autopkgtest-reboot-prepare has been called.
+          [Journal]
+          ForwardToConsole=yes
+    - path: /etc/zincati/config.d/99-config.toml
+      mode: 0644
+      contents:
+        inline: |
+          # Don't wait for any rollout window, update ASAP
+          [identity]
+          rollout_wariness = 0.0000
+          # Increase the frequency at which we check for updates
+          [agent.timing]
+          steady_interval_secs = 20

tests/kola/upgrade/extended/data/commonlib.sh

@@ -0,0 +1 @@
+../../../data/commonlib.sh

tests/kola/upgrade/extended/test.sh

@@ -0,0 +1,158 @@
+#!/bin/bash
+## kola:
+##   # - needs-internet: to pull updates
+##   tags: "needs-internet"
+##   # Extend the timeout since a lot of updates/reboots can happen.
+##   timeoutMin: 45
+##   # Only run this test when specifically requested.
+##   requiredTag: extended-upgrade
+
+set -eux -o pipefail
+
+. $KOLA_EXT_DATA/commonlib.sh
+
+# This test will attempt to test an upgrade from a given starting
+# point (assumed by the caller passing in a specific
+# `cosa kola run --build=x.y.z`) all the way to the latest build
+# that is staged to be released. The test is basic in that it
+# essentially tests 1) updates work 2) boot works.
+#
+# An example invocation for this test would look like:
+
+# ```
+# cosa buildfetch --stream=next --build=34.20210904.1.0 --artifact=qemu
+# cosa decompress --build=34.20210904.1.0
+# cosa kola run --build=34.20210904.1.0 --tag extended-upgrade
+# ```
+#
+# You can monitor the progress from the console and journal:
+#   - everything:
+#       - tail -f tmp/kola/ext.config.upgrade.extended/*/console.txt
+#   - major events:
+#       - tail -f tmp/kola/ext.config.upgrade.extended/*/journal.txt | grep --color -i 'ok reached version'
+#
+# For convenience, here is a list of the earliest releases on each
+# stream/architecture:
+#
+# stable
+#   - x86_64  31.20200108.3.0
+#   - aarch64 34.20210821.3.0
+#   - s390x   36.20220618.3.1
+# testing
+#   - x86_64  30.20190716.1
+#   - aarch64 34.20210904.2.0
+#   - s390x   36.20220618.2.0
+# next
+#   - x86_64  32.20200416.1.0
+#   - aarch64 34.20210904.1.0
+#   - s390x   36.20220618.1.1
+
+. /etc/os-release # for $VERSION_ID
+
+# delete the disabling of updates that was done by the test framework
+if [ -f /etc/zincati/config.d/90-disable-auto-updates.toml ]; then
+    rm -f /etc/zincati/config.d/90-disable-auto-updates.toml
+    systemctl restart zincati
+fi
+
+version=$(rpm-ostree status  --json | jq -r '.deployments[0].version')
+stream=$(rpm-ostree status  --json | jq -r '.deployments[0]["base-commit-meta"]["fedora-coreos.stream"]')
+
+test -f /srv/releases.json || \
+    curl -L "https://builds.coreos.fedoraproject.org/prod/streams/${stream}/releases.json" > /srv/releases.json
+test -f /srv/builds.json || \
+    curl -L "https://builds.coreos.fedoraproject.org/prod/streams/${stream}/builds/builds.json" > /srv/builds.json
+
+last_release=$(jq -r .releases[-1].version /srv/releases.json)
+target_version=$(jq -r .builds[0].id /srv/builds.json)
+
+grab-gpg-keys() {
+    # For older FCOS we had an issue where when we tried to pull the
+    # commits from the repo it would fail if we were on N-2 because
+    # the newer commits would be signed with a key the old OS didn't
+    # know anything about. We applied a workaround in newer releases,
+    # so this workaround should be limited to zincati older than v0.0.24
+    # https://github.com/coreos/fedora-coreos-tracker/issues/749
+    max_version=${target_version:0:2} # i.e. 36, 37, 38, etc..
+    for ver in $(seq $VERSION_ID $max_version); do
+        test -e "/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-${ver}-primary" && continue
+        curl -L "https://src.fedoraproject.org/rpms/fedora-repos/raw/rawhide/f/RPM-GPG-KEY-fedora-${ver}-primary" | \
+            sudo tee "/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-${ver}-primary"
+        sudo chcon -v --reference="/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-${VERSION_ID}-primary" "/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-${ver}-primary"
+    done
+}
+
+fix-update-url() {
+    # We switched to non stg URL in zincati v0.0.10 [1]. For older clients
+    # we need to update the runtime configuration of zincati to get past the problem.
+    # [1] https://github.com/coreos/zincati/commit/1d73801ccd015cdce89f082cb1eeb9b4b8335760
+    cat <<'EOF' > /run/zincati/config.d/50-fedora-coreos-cincinnati.toml
+[cincinnati]
+base_url= "https://updates.coreos.fedoraproject.org"
+EOF
+    systemctl restart zincati
+}
+
+ok "Reached version: $version"
+
+# Are we all the way at the desired target version?
+# If so then we can exit with success!
+if vereq $version $target_version; then
+    ok "Fully upgraded to $target_version"
+    bootupctl status
+    exit 0
+fi
+
+# Apply workarounds based on the current version of the system.
+#
+# First release on each stream with new enough zincati for updates stg.fedoraprojec.org
+# - 31.20200505.3.0
+# - 31.20200505.2.0
+# - 32.20200505.1.0
+#
+# First release with new enough zincati with workaround for N-2 gpg key issue
+# - 35.20211119.3.0
+# - 35.20211119.2.0
+# - 35.20211119.1.0
+#
+case "$stream" in
+    'next')
+        verlt $version '35.20211119.1.0' && grab-gpg-keys
+        verlt $version '31.20200505.1.0' && fix-update-url
+        ;;
+    'testing')
+        verlt $version '35.20211119.2.0' && grab-gpg-keys
+        verlt $version '31.20200505.2.0' && fix-update-url
+        ;;
+    'stable')
+        verlt $version '35.20211119.3.0' && grab-gpg-keys
+        verlt $version '31.20200505.3.0' && fix-update-url
+        ;;
+    *) fatal "unexpected stream: $stream";;
+esac
+
+# If we have made it all the way to the last release then
+# we have one more test. We'll now rebase to the target
+# version, which should be in the compose OSTree repo.
+if vereq $version $last_release; then
+    systemctl stop zincati
+    rpm-ostree rebase fedora-compose: $target_version
+    /tmp/autopkgtest-reboot reboot # execute the reboot
+    sleep infinity
+fi
+
+# Watch the Zincati logs to see if it got a lead on a new update.
+# Timeout after some time if no update. Unset pipefail since the
+# journalctl -f will give a bad exit code when grep exits early.
+set +o pipefail
+cmd="journalctl -b 0 -f --no-tail -u zincati.service"
+if ! timeout 90s $cmd | grep --max-count=1 'proceeding to stage it'; then
+    # No update initiated within timeout; let's error.
+    fatal "Updating the system stalled out on version: $version"
+fi
+set -o pipefail
+
+
+# OK update has been initiated, prepare for reboot and sleep
+/tmp/autopkgtest-reboot-prepare reboot
+sleep infinity