Merge pull request #105 from juju-solutions/feature/ha-flag

Feature/ha flag

.gitignore

@@ -9,6 +9,6 @@ build/
 /.coverage
 /.cover/
 matrix.log
-glitch_plan*.yaml
+chaos_plan*.yaml
 .tox
 .cache

matrix/main.py

@@ -5,6 +5,7 @@
 from pathlib import Path
 from pkg_resources import resource_filename
 import os
+import yaml
 import sys
 
 from .bus import Bus, set_default_bus
@@ -44,6 +45,39 @@ def configLogging(options):
         fil.update_selections(options.log_filter)
 
 
+def add_bundle_opts(options, parser):
+    """
+    Add arguments from a 'matrix' section of the bundle's tests.yaml
+    to our options object.
+
+    Don't allow a bundle to override anything that we've set from the
+    command line (protects from potential mischief).
+
+    """
+    test_yaml_file = Path(options.path, 'tests', 'test.yaml')
+    if not test_yaml_file.exists():
+        return options
+
+    with test_yaml_file.open() as test_yaml:
+        test_yaml = test_yaml.read()
+        test_yaml = yaml.load(test_yaml)
+
+    bundle_opts = test_yaml.get('matrix')
+
+    # Run things through the parser to valiate that our arguments are
+    # well formed.
+    parser.parse_args(bundle_opts)
+
+    for key in bundle_opts:
+        # If a key is not in options, or the key is set to a default
+        # value, add the value from the bundle to our options.
+        if not hasattr(options, key) or \
+           getattr(options, key) == parser.get_default(key):
+            setattr(options, key, bundle_opts[key])
+
+    return options
+
+
 def setup(matrix, args=None):
     parser = argparse.ArgumentParser(
         formatter_class=utils.ParagraphDescriptionFormatter,
@@ -142,15 +176,23 @@ def setup(matrix, args=None):
                               "mode.".format(
                                   RAW_TIMEOUT,
                                   TUI_TIMEOUT or "no timeout")))
+    parser.add_argument("-H", "--ha", action='store_true',
+                        help=("Treat this bundle as a 'high availabilty' "
+                              "bundle. This means that tests that gate on "
+                              "'ha_only' will gate on this bundle."))
+
     options = parser.parse_args(args, namespace=matrix)
+    options = add_bundle_opts(options, parser)
+
+    if not (options.path.is_dir() and (options.path / 'bundle.yaml').exists()):
+        parser.error('Invalid bundle directory: %s' % options.path)
+
     # Set default timeouts
     if options.timeout is None:
         if options.skin == 'raw':
             options.timeout = RAW_TIMEOUT
         if options.skin == 'tui':
             options.timeout = TUI_TIMEOUT  # None
-    if not (options.path.is_dir() and (options.path / 'bundle.yaml').exists()):
-        parser.error('Invalid bundle directory: %s' % options.path)
 
     configLogging(options)
     return options

matrix/matrix.yaml

@@ -19,7 +19,7 @@
       "periodic": 5
       "do":
         "task": matrix.tasks.health
-      "gating": false
+      "gating": ha_only
       "until": chaos.complete
     - "after": health.status.healthy
       "do":
@@ -28,4 +28,4 @@
     - "after": health.status.healthy
       "do":
         "task": matrix.tasks.chaos
-      "gating": false
+      "gating": ha_only

matrix/model.py

@@ -96,7 +96,7 @@ def __str__(self):
 class Task:
     command = attr.ib(convert=str)
     args = attr.ib(default=attr.Factory(dict))
-    gating = attr.ib(default=True, convert=bool)
+    gating = attr.ib(default=True)
 
     @property
     def name(self):

matrix/rules.py

@@ -325,7 +325,7 @@ def load_suite(self):
         if exceptions:
             for t, e in exceptions:
                 if type(e) is model.TestFailure:
-                    if e.task.gating is True:
+                    if utils.should_gate(context=self, task=e.task):
                         log.error(
                             "Setting exit code 1 due to gating TestFailure.")
                         self.exit_code = 1

matrix/tasks/chaos/main.py

@@ -5,6 +5,7 @@
 from pathlib import Path
 
 from .actions import Actions
+from matrix import utils
 from matrix.model import TestFailure
 from .plan import generate_plan, validate_plan
 from .selectors import Selectors
@@ -151,9 +152,8 @@ class NoObjects(Exception):
             rule.log.error(e)
             continue
 
-        if errors and task.gating:
-            raise TestFailure(
-                task, "Exceptions were raised during chaos run.")
+        if errors and utils.should_gate(context=context, task=task):
+            raise TestFailure(task, "Exceptions were raised during chaos run.")
 
         context.bus.dispatch(
             origin="chaos",

matrix/tasks/health.py

@@ -1,4 +1,5 @@
 from datetime import timedelta, datetime, timezone
+from matrix import utils
 from matrix.model import TestFailure
 
 
@@ -55,7 +56,7 @@
         _log = rule.log.info
     _log("Health check: %s", result)
 
-    if result == 'unhealthy' and task.gating:
+    if result == 'unhealthy' and utils.should_gate(context=context, task=task):
         rule.log.error("Raising TestFailure due to unhealthy status")
         raise TestFailure(task, 'Health state was unhealthy')
 

matrix/utils.py

@@ -241,6 +241,19 @@ def translate_ansi_colors(entity):
             "Is it installed in your environment?")
 
 
+def should_gate(context, task):
+    """
+    Determine whether or not we should "gate" (raise an error) on
+    failure, given a specific task, in a specific context.
+
+    """
+    if task.gating is True:
+        return True
+    if task.gating == 'ha_only' and context.ha is True:
+        return True
+    return False
+
+
 @contextmanager
 def new_event_loop():
     old_loop = asyncio.get_event_loop()

tests/basic_bundle/tests/test.yaml

@@ -0,0 +1,2 @@
+matrix:
+  model_prefix: matrixtest

tests/functional/gating_test.py

@@ -32,3 +32,16 @@ def test_turn_off_gating(self):
         proc = subprocess.run(self.cmd + [test], check=False, timeout=60)
         self.assertEqual(proc.returncode, 0)
         self.check_artifacts(1)  # log
+
+    def test_gate_on_ha(self):
+        test = 'tests/test_ha_gating.matrix'
+        proc = subprocess.run(
+            self.cmd + ['--ha', test], check=False, timeout=60)
+        self.assertEqual(proc.returncode, 1)
+        self.check_artifacts(1)  # log
+
+    def test_dont_gate_non_ha(self):
+        test = 'tests/test_ha_gating.matrix'
+        proc = subprocess.run(self.cmd + [test], check=False, timeout=60)
+        self.assertEqual(proc.returncode, 0)
+        self.check_artifacts(1)  # log

tests/test_config.py

@@ -0,0 +1,31 @@
+import mock
+import unittest
+
+from matrix import main
+
+
+class TestConfig(unittest.TestCase):
+
+    def test_add_bundle_opts(self):
+        """
+        _test_add_bundle_opts_
+
+        This test requires that we have a test.yaml in
+        tests/basic_bundle/test.yaml, with a model_prefix equal to
+        matrixtest.
+
+        """
+        options = mock.Mock()
+        parser = mock.Mock()
+        options.path = 'tests/basic_bundle'
+        options.model_prefix = 'matrix'  # default
+        parser.get_default.return_value = 'matrix'
+
+        # Verify that we can override a default value
+        options = main.add_bundle_opts(options, parser)
+        self.assertEqual(options.model_prefix, 'matrixtest')
+
+        # Verify that we can't override a non default value
+        options.model_prefix = 'bar'
+        options = main.add_bundle_opts(options, parser)
+        self.assertEqual(options.model_prefix, 'bar')

tests/test_ha_gating.matrix

@@ -0,0 +1,9 @@
+#!/usr/bin/env matrix
+"tests":
+- "name": Verify that we can turn gating off.
+  "description": >
+    This test succeeds if matrix exits with a zero exit code.
+  "rules":
+    - "do":
+        "task": matrix.tasks.fail
+      "gating": ha_only

tests/test_utils.py

@@ -0,0 +1,24 @@
+import unittest
+import mock
+
+from matrix import utils
+
+
+class TestUtils(unittest.TestCase):
+    def test_should_gate(self):
+        context = mock.Mock()
+        task = mock.Mock()
+
+        task.gating = True
+        self.assertTrue(utils.should_gate(context, task))
+
+        task.gating = 'ha_only'
+        context.ha = False
+        self.assertFalse(utils.should_gate(context, task))
+
+        task.gating = 'ha_only'
+        context.ha = True
+        self.assertTrue(utils.should_gate(context, task))
+
+        task.gating = False
+        self.assertFalse(utils.should_gate(context, task))