Add test for series upgrade

jobs/integration/conftest.py

@@ -79,6 +79,14 @@ def pytest_addoption(parser):
         help="Snap channel to install snapcore from",
     )
 
+    # Set when performing series upgrade tests
+    parser.addoption(
+        "--is-series-upgrade",
+        action="store_true",
+        default=False,
+        help="This test should perform a series upgrade",
+    )
+
 
 class Tools:
     """ Utility class for accessing juju related tools
@@ -91,6 +99,7 @@ def __init__(self, request):
         self.series = request.config.getoption("--series")
         self.cloud = request.config.getoption("--cloud")
         self.connection = f"{self.controller_name}:{self.model_name}"
+        self.is_series_upgrade = request.config.getoption('--is-series-upgrade')
 
     async def run(self, *cmd):
         proc = await asyncio.create_subprocess_shell(

jobs/integration/utils.py

@@ -6,13 +6,21 @@
 import time
 import traceback
 
+from cilib import log
 from contextlib import contextmanager
 from juju.controller import Controller
 from juju.errors import JujuError
 from subprocess import check_output, check_call
 import click
 
 
+# note: we can't upgrade to focal until after it's released
+SERIES_ORDER = [
+    'xenial',
+    'bionic',
+]
+
+
 def tracefunc(frame, event, arg):
     if event != "call":
         return
@@ -428,3 +436,53 @@ async def validate_storage_class(model, sc_name, test_name):
         (master, "pvc", pvcs),
         timeout_msg="Unable to remove {} test pvcs".format(test_name),
     )
+
+
+def _units(machine):
+    return [unit for unit in machine.model.units if unit.machine.id == machine.id]
+
+
+async def wait_for_status(workload_status, units):
+    if not isinstance(units, (list, tuple)):
+        units = [units]
+    log.info(f'waiting for {workload_status} status on {", ".join(units)}')
+    model = units[0].model
+    try:
+        await model.block_until(lambda: all(unit.workload_status == workload_status for unit in units),
+                                timeout=120)
+    except asyncio.TimeoutError as e:
+        unmatched_units = [f'{unit.name}={unit.workload_status}'
+                           for unit in units if unit.workload_status != workload_status]
+        raise AssertionError(f'Units with unexpected status: {",".join(unmatched_units)}') from e
+
+
+async def prep_series_upgrade(machine, new_series, tools):
+    log.info(f'preparing series upgrade for machine {machine.id}')
+    await tools.run('juju', 'upgrade-series', '--yes', '-m', tools.connection,
+                    machine.id, 'prepare', new_series)
+    await wait_for_status('blocked', _units(machine))
+
+
+async def do_series_upgrade(machine):
+    file_name = '/etc/apt/apt.conf.d/50unattended-upgrades'
+    option = '--force-confdef'
+    log.info(f'doing series upgrade for machine {machine.id}')
+    await machine.ssh(f"""
+        if ! grep -q -- '{option}' {file_name}; then
+          echo 'DPkg::options {{ "{option}"; }};' | sudo tee -a {file_name}
+        fi
+        sudo DEBIAN_FRONTEND=noninteractive do-release-upgrade -f DistUpgradeViewNonInteractive
+    """)
+    log.info(f'rebooting machine {machine.id}')
+    try:
+        await machine.ssh('sudo reboot && exit')
+    except JujuError:
+        # We actually expect this to "fail" because the reboot closes the session prematurely.
+        pass
+
+
+async def finish_series_upgrade(machine, tools):
+    log.info(f'completing series upgrade for machine {machine.id}')
+    await tools.run('juju', 'upgrade-series', '--yes', '-m', tools.connection,
+                    machine.id, 'complete')
+    await wait_for_status('active', _units(machine))

jobs/integration/validation.py

@@ -28,6 +28,10 @@
     tracefunc,
     is_localhost,
     validate_storage_class,
+    SERIES_ORDER,
+    prep_series_upgrade,
+    do_series_upgrade,
+    finish_series_upgrade,
 )
 import sys
 
@@ -248,7 +252,7 @@ async def test_auth_file_propagation(model):
 
 @pytest.mark.asyncio
 @pytest.mark.flaky
-async def test_status_messages(model, tools):
+async def test_status_messages(model):
     """ Validate that the status messages are correct. """
     expected_messages = {
         "kubernetes-master": "Kubernetes master running.",
@@ -1875,3 +1879,21 @@ async def cleanup():
     await cleanup()
 
 
+@pytest.mark.asyncio
+async def test_series_upgrade(model, tools):
+    if not tools.is_series_upgrade:
+        pytest.skip()
+    k8s_master_0 = model.applications['kubernetes-master'].units[0]
+    old_series = k8s_master_0.machine.series
+    try:
+        new_series = SERIES_ORDER[SERIES_ORDER.index(old_series) + 1]
+    except IndexError:
+        pytest.skip("no supported series to upgrade to")
+    except ValueError:
+        pytest.skip("unrecognized series to upgrade from: {old_series}")
+    for machine in model.machines.values():
+        prep_series_upgrade(machine, new_series, tools)
+        do_series_upgrade(machine)
+        finish_series_upgrade(machine, tools)
+        assert machine.series == new_series
+    test_status_messages(model)

jobs/validate.yaml

@@ -109,6 +109,32 @@
             venv/bin/ogc --spec jobs/validate/snapd-upgrade-spec.yml
 
 
+- job:
+    name: 'validate-ck-series-upgrade'
+    node: runner-validate
+    description: |
+      Validates CK machine series upgrades.
+    project-type: freestyle
+    scm:
+      - k8s-jenkins-jenkaas
+    wrappers:
+      - default-job-wrapper
+      - ci-creds
+    triggers:
+        - timed: "@weekly"
+    properties:
+      - build-discarder:
+          num-to-keep: 7
+    builders:
+      - set-env:
+          JOB_SPEC_DIR: "jobs/validate"
+      - run-venv:
+          JOB_SPEC_DIR: "jobs/validate"
+          COMMAND: |
+            venv/bin/python -m pip install https://github.com/battlemidget/ogc/archive/master.zip
+            venv/bin/ogc --spec jobs/validate/series-upgrade-spec.yml
+
+
 # ADDONS --------------------------------------------------------------------- #
 
 

jobs/validate/series-upgrade-spec.yml

@@ -0,0 +1,84 @@
+meta:
+  name: Verify CK with machine series upgrade
+  description: |
+    Verifies that CK minor upgrades from previous stable to upcoming edge passes integration tests
+  mkdocs:
+    destination:
+      - "validations/ck/minor-upgrade.md"
+
+matrix:
+  snap_version:
+    - 1.18/stable
+  series:
+    - xenial
+  channel:
+    - stable
+  arch:
+    - amd64
+
+plan:
+  env:
+    - JUJU_DEPLOY_BUNDLE=cs:~containers/charmed-kubernetes
+    - JUJU_DEPLOY_CHANNEL=$CHANNEL
+    - JUJU_CLOUD=aws/us-east-2
+    - JUJU_CONTROLLER=validate-ck-series-upgrade
+    - JUJU_MODEL=validate-series-upgrade
+
+  pre-execute: |
+    #!/bin/bash
+    . $WORKSPACE/cilib.sh
+
+    setup_env
+
+    juju bootstrap $JUJU_CLOUD $JUJU_CONTROLLER \
+         -d $JUJU_MODEL \
+         --bootstrap-series $SERIES \
+         --force \
+         --bootstrap-constraints arch=$ARCH \
+         --model-default test-mode=true \
+         --model-default resource-tags=owner=k8sci \
+         --model-default image-stream=daily
+
+    tee overlay.yaml <<EOF> /dev/null
+    series: $SERIES
+    applications:
+      kubernetes-master:
+        options:
+          channel: $SNAP_VERSION
+      kubernetes-worker:
+        options:
+          channel: $SNAP_VERSION
+    EOF
+
+    juju deploy -m $JUJU_CONTROLLER:$JUJU_MODEL \
+          --overlay overlay.yaml \
+          --force \
+          --channel $JUJU_DEPLOY_CHANNEL $JUJU_DEPLOY_BUNDLE
+
+    timeout 45m juju-wait -e $JUJU_CONTROLLER:$JUJU_MODEL -w
+
+  execute: |
+    #!/bin/bash
+    set -eu
+
+    . $WORKSPACE/cilib.sh
+
+    inject_env
+
+    timeout 2h pytest $WORKSPACE/jobs/integration/validation.py::test_series_upgrade \
+           --html=$OGC_JOB_WORKDIR/report.html --self-contained-html \
+           --is-series-upgrade \
+           --cloud "$JUJU_CLOUD" \
+           --model "$JUJU_MODEL" \
+           --controller "$JUJU_CONTROLLER"
+
+  post-execute: |
+    #!/bin/bash
+    . $WORKSPACE/cilib.sh
+
+    inject_env
+
+    collect_env
+
+    teardown_env
+