fix(install): Inject YAML nodes to fix disabling cassandra. (#1540)

Rewrote YAML mutator so it uses parse tree rather than regex searching
for more robustness.

dev/bootstrap_dev.sh

@@ -237,6 +237,16 @@ if [[ "$CONFIRMED_GITHUB_REPOSITORY_OWNER" != "none" ]]; then
       --github_user $CONFIRMED_GITHUB_REPOSITORY_OWNER
 fi
 
+# Prepare spinnaker-local.yml
+if [[ ! -f $HOME/.spinnaker/spinnaker-local.yml ]]; then
+    # This has a side effect in which it will produce a spinnaker-local if
+    # it did not already exist. Since there is no "bogus" microservice, this
+    # will not actually do anything other than produce the spinnaker-local.
+    # The default spinnaker-local will be configured for this machine as
+    # a better starting point than otherwise.
+    ./spinnaker/dev/stop_dev.sh bogus >& /dev/null || true
+fi
+
 # Some dependencies of Deck rely on Bower to manage their dependencies. Bower
 # annoyingly prompts the user to collect some stats, so this disables that.
 echo "{\"interactive\":false}" > ~/.bowerrc

dev/dev_runner.py

@@ -169,8 +169,9 @@ def maybe_generate_clean_user_local():
     change_path = os.path.join(os.path.dirname(os.path.dirname(__file__)),
                                'install', 'change_cassandra.sh')
     got = run_quick(change_path
-              + ' --echo=inMemory --front50=gcs'
-              + ' --change_defaults=false --change_local=true')
+                    + ' --echo=inMemory --front50=gcs'
+                    + ' --change_defaults=false --change_local=true',
+                    echo=False)
 
   def __init__(self, installation_parameters=None):
     self.maybe_generate_clean_user_local()

pylib/spinnaker/change_cassandra.py

@@ -54,10 +54,14 @@
 
 FRONT50_CHOICES = ['cassandra', 's3', 'gcs', 'redis', 'azs']
 
-_ECHO_KEYS = ['echo.cassandra.enabled', 'echo.inMemory.enabled']
-_FRONT50_KEYS = ['front50.cassandra.enabled', 'front50.redis.enabled',
-                 'front50.s3.enabled', 'front50.gcs.enabled',
-                 'front50.storage_bucket', 'front50.azs.enabled']
+_ECHO_KEYS = ['services.echo.cassandra.enabled',
+              'services.echo.inMemory.enabled']
+_FRONT50_KEYS = ['services.front50.cassandra.enabled',
+                 'services.front50.redis.enabled',
+                 'services.front50.s3.enabled',
+                 'services.front50.gcs.enabled',
+                 'services.front50.storage_bucket',
+                 'services.front50.azs.enabled']
 
 SPINNAKER_INSTALLED_PATH = '/opt/spinnaker/cassandra/SPINNAKER_INSTALLED_CASSANDRA'
 SPINNAKER_DISABLED_PATH = '/opt/spinnaker/cassandra/SPINNAKER_DISABLED_CASSANDRA'
@@ -115,7 +119,7 @@ def __init__(self, options):
                          }}
     if options.bucket:
         config['front50']['storage_bucket'] = options.bucket
-    self.__bindings.import_dict(config)
+    self.__bindings.import_dict({'services': config})
 
   def disable_cassandra(self):
     if os.path.exists(SPINNAKER_INSTALLED_PATH):

pylib/spinnaker/yaml_util.py

@@ -168,7 +168,7 @@ def __get_flat_keys(self, container):
 
 
   @staticmethod
-  def update_yml_source(path, update_dict):
+  def update_yml_source(path, update_dict, add_new_nodes=True):
     """Update the yaml source at the path according to the update dict.
 
     All the previous bindings not in the update dict remain unchanged.
@@ -178,6 +178,8 @@ def update_yml_source(path, update_dict):
       path [string]: Path to a yaml source file.
       update_dict [dict]: Nested dictionary corresponding to
           nested yaml properties, keyed by strings.
+      add_new_nodes [boolean]: If true, add nodes in update_dict that
+          were not in the original. Otherwise raise a KeyError.
     """
     bindings = YamlBindings()
     bindings.import_dict(update_dict)
@@ -186,13 +188,96 @@ def update_yml_source(path, update_dict):
     with open(path, 'r') as source_file:
       source = source_file.read()
       for prop in updated_keys:
-        source = bindings.transform_yaml_source(source, prop)
+        source = bindings.transform_yaml_source(source, prop,
+                                                add_new_nodes=add_new_nodes)
 
     with open(path, 'w') as source_file:
       source_file.write(source)
 
+  def find_yaml_context(self, source, root_node, full_key, raise_if_not_found):
+    """Given a yaml file and full key, find the span of the value for the key.
 
-  def transform_yaml_source(self, source, key):
+    If the key does not exist, then return how we should modify the file
+    around the insertion point so that the context is there.
+
+    Args:
+      source: [string] The YAML source text.
+      root_node: [yaml.nodes.MappingNode]  The composed yaml tree
+      full_key: [string] Dot-delimited path whose value we're looking for
+      raise_if_not_found: [boolean] Whether to raise a KeyError or return
+         additional context if key isnt found.
+
+    Returns:
+      context, span
+
+      where:
+          context:[string, string)]: Text to append before and after the cut.
+          span: [(int, int)]: The start/end range of source with existing value
+                              to cut.
+    """
+    parts = full_key.split('.')
+    if not isinstance(root_node, yaml.nodes.MappingNode):
+      if not root_node and not raise_if_not_found:
+        return (self._make_missing_key_text('', parts), '\n'), (0, 0)
+      else:
+        raise ValueError(root_node.__class__.__name__ + ' is not a yaml node.')
+
+    closest_node = None
+    for key_index, key in enumerate(parts):
+      found = False
+      for node in root_node.value:
+        if node[0].value == key:
+          closest_node = node
+          found = True
+          break
+      if not found:
+        break
+      root_node = closest_node[1]
+
+    if closest_node is None:
+      if raise_if_not_found:
+        raise KeyError(full_key)
+      # Nothing matches, so stick this at the start of the file.
+      return (self._make_missing_key_text('', parts), '\n'), (0, 0)
+
+    span = (closest_node[1].start_mark.index,
+            closest_node[1].end_mark.index)
+    span_is_empty = span[0] == span[1]
+
+    if found:
+      # value of closest_node is what we are going to replace.
+      # There is still a space between the token and value we write.
+      return (' ' if span_is_empty else '', ''), span
+
+    if raise_if_not_found:
+      raise KeyError('.'.join(parts[0:key_index + 1]))
+
+    # We are going to add a new child. This is going to be indented equal
+    # to the current line if the value isnt empty, otherwise one more level.
+    line_start = closest_node[0].start_mark.index - 1
+    while line_start >= 0 and source[line_start] == ' ':
+      line_start -= 1
+
+    indent = ' ' * (closest_node[0].start_mark.index - line_start + 1)
+    key_text = self._make_missing_key_text(indent, parts[key_index:])
+    if span_is_empty:
+      key_text = '  ' + prefix
+    return (key_text, '\n' + indent), (span[0], span[0])
+
+  def _make_missing_key_text(self, indent, keys):
+    key_context = []
+    sep = ''
+    for depth, key in enumerate(keys):
+      key_context.append('{sep}{extra_indent}{key}:'
+                         .format(sep=sep,
+                                 extra_indent='  ' * depth,
+                                 key=key,
+                                 base_indent=indent))
+      sep = '\n' + indent
+    key_context.append(' ')
+    return ''.join(key_context)
+
+  def transform_yaml_source(self, source, key, add_new_nodes=True):
     """Transform the given yaml source so its value of key matches the binding.
 
     Has no effect if key is not among the bindings.
@@ -201,6 +286,8 @@ def transform_yaml_source(self, source, key):
     Args:
       source [string]: A YAML document
       key [string]: A key into the bindings.
+      add_new_nodes [boolean]: If true, add node for key if not already present.
+           Otherwise raise a KeyError.
 
     Returns:
       Transformed source with value of key replaced to match the bindings.
@@ -210,22 +297,6 @@ def transform_yaml_source(self, source, key):
     except KeyError:
       return source
 
-    parts = key.split('.')
-    offset = 0
-    s = source
-    for attr in parts:
-        match = re.search('^ *{attr}:(.*)'.format(attr=attr), s, re.MULTILINE)
-        if not match:
-            raise ValueError(
-                'Could not find {key}. Failed on {attr} at {offset}'
-                .format(key=key, attr=attr, offset=offset))
-        offset += match.start(0)
-        s = source[offset:]
-
-    offset -= match.start(0)
-    value_start = match.start(1) + offset
-    value_end = match.end(0) + offset
-
     if isinstance(value, basestring) and re.search('{[^}]*{', value):
       # Quote strings with nested {} yaml flows
       value = '"{0}"'.format(value)
@@ -234,10 +305,20 @@ def transform_yaml_source(self, source, key):
     if isinstance(value, bool):
       value = str(value).lower()
 
+    yaml_root = yaml.compose(source)
+    context, span = self.find_yaml_context(
+        source, yaml_root, key, not add_new_nodes)
+
+    text_before = context[0]
+    text_after = context[1]
+    start_cut = span[0]
+    end_cut = span[1]
     return ''.join([
-        source[0:value_start],
-        ' {value}'.format(value=value),
-        source[value_end:]
+        source[0:start_cut],
+        text_before,
+        '{value}'.format(value=value),
+        text_after,
+        source[end_cut:]
     ])
 
 

unittest/yaml_util_test.py

@@ -285,27 +285,22 @@ def test_concat_default(self):
   def test_transform_ok(self):
      bindings = YamlBindings()
      bindings.import_dict({'a': {'b': { 'space': 'WithSpace',
-                                        'nospace': 'WithoutSpace',
                                         'empty': 'Empty'}},
                            'x' : {'unique': True}})
      template = """
 a:
   b:
     space: {space}
-    nospace:{nospace}
     empty:{empty}
 unique:
   b:
      space: A
-     nospace:B
      empty:
 """
-     source = template.format(space='SPACE', nospace='NOSPACE', empty='')
-     expect = template.format(space='WithSpace',
-                              nospace=' WithoutSpace',
-                              empty=' Empty')
+     source = template.format(space='SPACE', empty='')
+     expect = template.format(space='WithSpace', empty=' Empty')
      got = source
-     for key in [ 'a.b.space', 'a.b.nospace', 'a.b.empty' ]:
+     for key in [ 'a.b.space', 'a.b.empty' ]:
        got = bindings.transform_yaml_source(got, key)
 
      self.assertEqual(expect, bindings.transform_yaml_source(expect, 'bogus'))
@@ -321,8 +316,8 @@ def test_transform_fail(self):
   b:
      child: Hello
 """
-     with self.assertRaises(ValueError):
-       bindings.transform_yaml_source(yaml, 'x.unique')
+     with self.assertRaises(KeyError):
+       bindings.transform_yaml_source(yaml, 'x.unique', add_new_nodes=False)
 
   def test_list(self):
      bindings = YamlBindings()
@@ -358,6 +353,21 @@ def test_write_bool(self):
 
     os.remove(temp_path)
 
+  def test_create_yml_source(self):
+    expect = {
+      'first': { 'child': 'FirstValue' },
+      'second': { 'child': True }
+    }
+    fd, temp_path = tempfile.mkstemp()
+    os.write(fd, "")
+    os.close(fd)
+    YamlBindings.update_yml_source(temp_path, expect)
+
+    comparison_bindings = YamlBindings()
+    comparison_bindings.import_path(temp_path)
+    self.assertEqual(expect, comparison_bindings.map)
+    os.remove(temp_path)
+
   def test_update_yml_source(self):
     yaml = """
 a: A
@@ -378,7 +388,10 @@ def test_update_yml_source(self):
       'b': 'Z',
       'd': {
         'child': {
-          'grandchild': 'xy'
+          'grandchild': 'xy',
+          'new_grandchild': {
+             'new_node': 'inserted'
+          }
         }
       },
       'e': 'AA'
@@ -389,11 +402,22 @@ def test_update_yml_source(self):
               'c': ['A','B'],
               'd': {
                 'child': {
-                  'grandchild': 'xy'
+                  'grandchild': 'xy',
+                  'new_grandchild': {
+                    'new_node': 'inserted'
+                  }
                 }
               },
               'e': 'AA'}
 
+    with self.assertRaises(KeyError):
+      YamlBindings.update_yml_source(
+          temp_path, update_dict, add_new_nodes=False)
+
+    # Reset the file
+    with open(temp_path, 'w') as fd:
+      fd.write(yaml)
+
     YamlBindings.update_yml_source(temp_path, update_dict)
 
     comparison_bindings = YamlBindings()