Initial upgrade tests (#44)

* Add initial upgrade tests

* Parameterize test cases

* Upgrade kubeapi-load-balancer when present

* Get some docs in there.

* Simplify upgrade charm test. Just upgrade every app in the model.

* Add timeout to wait_for_ready, misc cleanup

* fix missing await

* Update README to mention juju controller usage

* use channel='beta' for now

* Add test_deploy,py, start on microbot validator

* Test against edge channels instead of beta

* Dump model info on test failure

* fix incorrect comment, oops

tests/upgrade-tests/README.md

@@ -0,0 +1,27 @@
+# CDK upgrade tests
+
+## Install
+
+Dependencies can be installed by running:
+```
+./install-deps.sh
+```
+
+## Running tests
+
+This test suite assumes that a juju controller has already been bootstrapped.
+
+Select the juju controller you want to use:
+```
+juju switch my-controller
+```
+
+To run all tests:
+```
+pytest
+```
+
+To run tests from a single file:
+```
+pytest test_upgrade_charms.py
+```

tests/upgrade-tests/install-deps.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+set -eux
+
+# Installs dependencies needed for upgrade tests
+# Can be run again to upgrade dependencies.
+
+sudo pip3 install -U pytest pytest-asyncio asyncio_extras juju

tests/upgrade-tests/test_deploy.py

@@ -0,0 +1,18 @@
+import pytest
+from utils import temporary_model, wait_for_ready
+from validation import validate_all
+
+test_cases = [
+    # bundle                 # channel
+    ('kubernetes-core',      'edge'),
+    ('canonical-kubernetes', 'edge'),
+]
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize('bundle,channel', test_cases)
+async def test_deploy(bundle, channel):
+    async with temporary_model() as model:
+        await model.deploy(bundle, channel=channel)
+        await wait_for_ready(model)
+        await validate_all(model)

tests/upgrade-tests/test_upgrade_charms.py

@@ -0,0 +1,21 @@
+import pytest
+from utils import temporary_model, wait_for_ready
+from validation import validate_all
+
+test_cases = [
+    # bundle                 from_channel  to_channel
+    ('kubernetes-core',      'stable',     'edge'),
+    ('canonical-kubernetes', 'stable',     'edge'),
+]
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize('bundle,from_channel,to_channel', test_cases)
+async def test_upgrade_charms(bundle, from_channel, to_channel):
+    async with temporary_model() as model:
+        await model.deploy(bundle, channel=from_channel)
+        await wait_for_ready(model)
+        for app in model.applications.values():
+            await app.upgrade_charm(channel=to_channel)
+        await wait_for_ready(model)
+        await validate_all(model)

tests/upgrade-tests/test_upgrade_snaps.py

@@ -0,0 +1,35 @@
+import pytest
+from utils import temporary_model, wait_for_ready
+from validation import validate_all
+
+test_cases = [
+    # bundle                 charm_channel  from_channel  to_channel
+    ('kubernetes-core',      'edge',        '1.6/stable', '1.6/edge'),
+    ('kubernetes-core',      'edge',        '1.6/stable', '1.7/edge'),
+    ('canonical-kubernetes', 'edge',        '1.6/stable', '1.6/edge'),
+    ('canonical-kubernetes', 'edge',        '1.6/stable', '1.7/edge'),
+]
+
+
+async def set_snap_channel(model, channel):
+    master = model.applications['kubernetes-master']
+    await master.set_config({'channel': channel})
+    worker = model.applications['kubernetes-worker']
+    await worker.set_config({'channel': channel})
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize('bundle,charm_channel,from_channel,to_channel',
+                         test_cases)
+async def test_upgrade_snaps(bundle, charm_channel, from_channel, to_channel):
+    async with temporary_model() as model:
+        await model.deploy(bundle, channel=charm_channel)
+        await set_snap_channel(model, from_channel)
+        await wait_for_ready(model)
+        await set_snap_channel(model, to_channel)
+        for unit in model.applications['kubernetes-worker'].units:
+            action = await unit.run_action('upgrade')
+            await action.wait()
+            assert action.status == 'completed'
+        await wait_for_ready(model)
+        await validate_all(model)

tests/upgrade-tests/utils.py

@@ -0,0 +1,70 @@
+import asyncio
+import json
+import random
+import sys
+from asyncio_extras import async_contextmanager
+from async_generator import yield_
+from juju.controller import Controller
+
+
+def dump_model_info(model):
+    ''' Dumps information about the model to stdout '''
+    data = {
+        'applications': {k: v.data for k, v in model.applications.items()},
+        'units': {k: v.data for k, v in model.units.items()},
+        'machines': {k: v.data for k, v in model.machines.items()}
+    }
+    json.dump(data, sys.stdout, indent=2)
+
+
+@async_contextmanager
+async def temporary_model():
+    ''' Create and destroy a temporary Juju model named cdk-build-upgrade-*.
+
+    This is an async context, to be used within an `async with` statement.
+    '''
+    controller = Controller()
+    await controller.connect_current()
+    model_name = 'cdk-build-upgrade-%d' % random.randint(0, 10000)
+    model = await controller.add_model(model_name)
+    try:
+        await yield_(model)
+    except:
+        dump_model_info(model)
+        raise
+    finally:
+        await model.disconnect()
+        await controller.destroy_model(model.info.uuid)
+        await controller.disconnect()
+
+
+def assert_no_unit_errors(model):
+    for unit in model.units.values():
+        assert unit.data['workload-status']['current'] != 'error'
+
+
+def all_units_ready(model):
+    ''' Returns True if all units are 'active' and 'idle', False otherwise. '''
+    for unit in model.units.values():
+        if unit.data['workload-status']['current'] != 'active':
+            return False
+        if unit.data['agent-status']['current'] != 'idle':
+            return False
+    return True
+
+
+async def wait_for_ready(model):
+    ''' Wait until all units are 'active' and 'idle'. '''
+    # FIXME: We might need to wait for more than just unit status.
+    #
+    # Subordinate units, for example, don't come into existence until after the
+    # principal unit has settled.
+    #
+    # If you see problems where this didn't wait long enough, it's probably
+    # that.
+    loop = asyncio.get_event_loop()
+    deadline = loop.time() + 1800  # 30 minutes
+    while not all_units_ready(model):
+        assert_no_unit_errors(model)
+        assert loop.time() < deadline
+        await asyncio.sleep(1)

tests/upgrade-tests/validation.py

@@ -0,0 +1,28 @@
+from utils import assert_no_unit_errors
+
+
+async def validate_all(model):
+    validate_status_messages(model)
+    await validate_microbot(model)
+    assert_no_unit_errors(model)
+
+
+def validate_status_messages(model):
+    ''' Validate that the status messages are correct. '''
+    expected_messages = {
+        'kubernetes-master': 'Kubernetes master running.',
+        'kubernetes-worker': 'Kubernetes worker running.'
+    }
+    for app, message in expected_messages.items():
+        for unit in model.applications[app].units:
+            assert unit.data['workload-status']['message'] == message
+
+
+async def validate_microbot(model):
+    ''' Validate the microbot action '''
+    unit = model.applications['kubernetes-worker'].units[0]
+    action = await unit.run_action('microbot', replicas=3)
+    await action.wait()
+    assert action.status == 'completed'
+    # TODO: wait for pods running
+    # TODO: test that we can reach the ingress endpoint