diff --git a/dev/bootstrap_dev.sh b/dev/bootstrap_dev.sh index 414be7dca..46bbfaa7a 100755 --- a/dev/bootstrap_dev.sh +++ b/dev/bootstrap_dev.sh @@ -237,6 +237,16 @@ if [[ "$CONFIRMED_GITHUB_REPOSITORY_OWNER" != "none" ]]; then --github_user $CONFIRMED_GITHUB_REPOSITORY_OWNER fi +# Prepare spinnaker-local.yml +if [[ ! -f $HOME/.spinnaker/spinnaker-local.yml ]]; then + # This has a side effect in which it will produce a spinnaker-local if + # it did not already exist. Since there is no "bogus" microservice, this + # will not actually do anything other than produce the spinnaker-local. + # The default spinnaker-local will be configured for this machine as + # a better starting point than otherwise. + ./spinnaker/dev/stop_dev.sh bogus >& /dev/null || true +fi + # Some dependencies of Deck rely on Bower to manage their dependencies. Bower # annoyingly prompts the user to collect some stats, so this disables that. echo "{\"interactive\":false}" > ~/.bowerrc diff --git a/dev/dev_runner.py b/dev/dev_runner.py index 374f68f0a..79c206612 100755 --- a/dev/dev_runner.py +++ b/dev/dev_runner.py @@ -169,8 +169,9 @@ def maybe_generate_clean_user_local(): change_path = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'install', 'change_cassandra.sh') got = run_quick(change_path - + ' --echo=inMemory --front50=gcs' - + ' --change_defaults=false --change_local=true') + + ' --echo=inMemory --front50=gcs' + + ' --change_defaults=false --change_local=true', + echo=False) def __init__(self, installation_parameters=None): self.maybe_generate_clean_user_local() diff --git a/pylib/spinnaker/change_cassandra.py b/pylib/spinnaker/change_cassandra.py index 6ababa321..dd6768f5f 100644 --- a/pylib/spinnaker/change_cassandra.py +++ b/pylib/spinnaker/change_cassandra.py @@ -54,10 +54,14 @@ FRONT50_CHOICES = ['cassandra', 's3', 'gcs', 'redis', 'azs'] -_ECHO_KEYS = ['echo.cassandra.enabled', 'echo.inMemory.enabled'] -_FRONT50_KEYS = ['front50.cassandra.enabled', 'front50.redis.enabled', - 'front50.s3.enabled', 'front50.gcs.enabled', - 'front50.storage_bucket', 'front50.azs.enabled'] +_ECHO_KEYS = ['services.echo.cassandra.enabled', + 'services.echo.inMemory.enabled'] +_FRONT50_KEYS = ['services.front50.cassandra.enabled', + 'services.front50.redis.enabled', + 'services.front50.s3.enabled', + 'services.front50.gcs.enabled', + 'services.front50.storage_bucket', + 'services.front50.azs.enabled'] SPINNAKER_INSTALLED_PATH = '/opt/spinnaker/cassandra/SPINNAKER_INSTALLED_CASSANDRA' SPINNAKER_DISABLED_PATH = '/opt/spinnaker/cassandra/SPINNAKER_DISABLED_CASSANDRA' @@ -115,7 +119,7 @@ def __init__(self, options): }} if options.bucket: config['front50']['storage_bucket'] = options.bucket - self.__bindings.import_dict(config) + self.__bindings.import_dict({'services': config}) def disable_cassandra(self): if os.path.exists(SPINNAKER_INSTALLED_PATH): diff --git a/pylib/spinnaker/yaml_util.py b/pylib/spinnaker/yaml_util.py index b2024705a..22b227cdc 100644 --- a/pylib/spinnaker/yaml_util.py +++ b/pylib/spinnaker/yaml_util.py @@ -168,7 +168,7 @@ def __get_flat_keys(self, container): @staticmethod - def update_yml_source(path, update_dict): + def update_yml_source(path, update_dict, add_new_nodes=True): """Update the yaml source at the path according to the update dict. All the previous bindings not in the update dict remain unchanged. @@ -178,6 +178,8 @@ def update_yml_source(path, update_dict): path [string]: Path to a yaml source file. update_dict [dict]: Nested dictionary corresponding to nested yaml properties, keyed by strings. + add_new_nodes [boolean]: If true, add nodes in update_dict that + were not in the original. Otherwise raise a KeyError. """ bindings = YamlBindings() bindings.import_dict(update_dict) @@ -186,13 +188,96 @@ def update_yml_source(path, update_dict): with open(path, 'r') as source_file: source = source_file.read() for prop in updated_keys: - source = bindings.transform_yaml_source(source, prop) + source = bindings.transform_yaml_source(source, prop, + add_new_nodes=add_new_nodes) with open(path, 'w') as source_file: source_file.write(source) + def find_yaml_context(self, source, root_node, full_key, raise_if_not_found): + """Given a yaml file and full key, find the span of the value for the key. - def transform_yaml_source(self, source, key): + If the key does not exist, then return how we should modify the file + around the insertion point so that the context is there. + + Args: + source: [string] The YAML source text. + root_node: [yaml.nodes.MappingNode] The composed yaml tree + full_key: [string] Dot-delimited path whose value we're looking for + raise_if_not_found: [boolean] Whether to raise a KeyError or return + additional context if key isnt found. + + Returns: + context, span + + where: + context:[string, string)]: Text to append before and after the cut. + span: [(int, int)]: The start/end range of source with existing value + to cut. + """ + parts = full_key.split('.') + if not isinstance(root_node, yaml.nodes.MappingNode): + if not root_node and not raise_if_not_found: + return (self._make_missing_key_text('', parts), '\n'), (0, 0) + else: + raise ValueError(root_node.__class__.__name__ + ' is not a yaml node.') + + closest_node = None + for key_index, key in enumerate(parts): + found = False + for node in root_node.value: + if node[0].value == key: + closest_node = node + found = True + break + if not found: + break + root_node = closest_node[1] + + if closest_node is None: + if raise_if_not_found: + raise KeyError(full_key) + # Nothing matches, so stick this at the start of the file. + return (self._make_missing_key_text('', parts), '\n'), (0, 0) + + span = (closest_node[1].start_mark.index, + closest_node[1].end_mark.index) + span_is_empty = span[0] == span[1] + + if found: + # value of closest_node is what we are going to replace. + # There is still a space between the token and value we write. + return (' ' if span_is_empty else '', ''), span + + if raise_if_not_found: + raise KeyError('.'.join(parts[0:key_index + 1])) + + # We are going to add a new child. This is going to be indented equal + # to the current line if the value isnt empty, otherwise one more level. + line_start = closest_node[0].start_mark.index - 1 + while line_start >= 0 and source[line_start] == ' ': + line_start -= 1 + + indent = ' ' * (closest_node[0].start_mark.index - line_start + 1) + key_text = self._make_missing_key_text(indent, parts[key_index:]) + if span_is_empty: + key_text = ' ' + prefix + return (key_text, '\n' + indent), (span[0], span[0]) + + def _make_missing_key_text(self, indent, keys): + key_context = [] + sep = '' + for depth, key in enumerate(keys): + key_context.append('{sep}{extra_indent}{key}:' + .format(sep=sep, + extra_indent=' ' * depth, + key=key, + base_indent=indent)) + sep = '\n' + indent + key_context.append(' ') + return ''.join(key_context) + + def transform_yaml_source(self, source, key, add_new_nodes=True): """Transform the given yaml source so its value of key matches the binding. Has no effect if key is not among the bindings. @@ -201,6 +286,8 @@ def transform_yaml_source(self, source, key): Args: source [string]: A YAML document key [string]: A key into the bindings. + add_new_nodes [boolean]: If true, add node for key if not already present. + Otherwise raise a KeyError. Returns: Transformed source with value of key replaced to match the bindings. @@ -210,22 +297,6 @@ def transform_yaml_source(self, source, key): except KeyError: return source - parts = key.split('.') - offset = 0 - s = source - for attr in parts: - match = re.search('^ *{attr}:(.*)'.format(attr=attr), s, re.MULTILINE) - if not match: - raise ValueError( - 'Could not find {key}. Failed on {attr} at {offset}' - .format(key=key, attr=attr, offset=offset)) - offset += match.start(0) - s = source[offset:] - - offset -= match.start(0) - value_start = match.start(1) + offset - value_end = match.end(0) + offset - if isinstance(value, basestring) and re.search('{[^}]*{', value): # Quote strings with nested {} yaml flows value = '"{0}"'.format(value) @@ -234,10 +305,20 @@ def transform_yaml_source(self, source, key): if isinstance(value, bool): value = str(value).lower() + yaml_root = yaml.compose(source) + context, span = self.find_yaml_context( + source, yaml_root, key, not add_new_nodes) + + text_before = context[0] + text_after = context[1] + start_cut = span[0] + end_cut = span[1] return ''.join([ - source[0:value_start], - ' {value}'.format(value=value), - source[value_end:] + source[0:start_cut], + text_before, + '{value}'.format(value=value), + text_after, + source[end_cut:] ]) diff --git a/unittest/yaml_util_test.py b/unittest/yaml_util_test.py index 1c35d2176..39f100219 100644 --- a/unittest/yaml_util_test.py +++ b/unittest/yaml_util_test.py @@ -285,27 +285,22 @@ def test_concat_default(self): def test_transform_ok(self): bindings = YamlBindings() bindings.import_dict({'a': {'b': { 'space': 'WithSpace', - 'nospace': 'WithoutSpace', 'empty': 'Empty'}}, 'x' : {'unique': True}}) template = """ a: b: space: {space} - nospace:{nospace} empty:{empty} unique: b: space: A - nospace:B empty: """ - source = template.format(space='SPACE', nospace='NOSPACE', empty='') - expect = template.format(space='WithSpace', - nospace=' WithoutSpace', - empty=' Empty') + source = template.format(space='SPACE', empty='') + expect = template.format(space='WithSpace', empty=' Empty') got = source - for key in [ 'a.b.space', 'a.b.nospace', 'a.b.empty' ]: + for key in [ 'a.b.space', 'a.b.empty' ]: got = bindings.transform_yaml_source(got, key) self.assertEqual(expect, bindings.transform_yaml_source(expect, 'bogus')) @@ -321,8 +316,8 @@ def test_transform_fail(self): b: child: Hello """ - with self.assertRaises(ValueError): - bindings.transform_yaml_source(yaml, 'x.unique') + with self.assertRaises(KeyError): + bindings.transform_yaml_source(yaml, 'x.unique', add_new_nodes=False) def test_list(self): bindings = YamlBindings() @@ -358,6 +353,21 @@ def test_write_bool(self): os.remove(temp_path) + def test_create_yml_source(self): + expect = { + 'first': { 'child': 'FirstValue' }, + 'second': { 'child': True } + } + fd, temp_path = tempfile.mkstemp() + os.write(fd, "") + os.close(fd) + YamlBindings.update_yml_source(temp_path, expect) + + comparison_bindings = YamlBindings() + comparison_bindings.import_path(temp_path) + self.assertEqual(expect, comparison_bindings.map) + os.remove(temp_path) + def test_update_yml_source(self): yaml = """ a: A @@ -378,7 +388,10 @@ def test_update_yml_source(self): 'b': 'Z', 'd': { 'child': { - 'grandchild': 'xy' + 'grandchild': 'xy', + 'new_grandchild': { + 'new_node': 'inserted' + } } }, 'e': 'AA' @@ -389,11 +402,22 @@ def test_update_yml_source(self): 'c': ['A','B'], 'd': { 'child': { - 'grandchild': 'xy' + 'grandchild': 'xy', + 'new_grandchild': { + 'new_node': 'inserted' + } } }, 'e': 'AA'} + with self.assertRaises(KeyError): + YamlBindings.update_yml_source( + temp_path, update_dict, add_new_nodes=False) + + # Reset the file + with open(temp_path, 'w') as fd: + fd.write(yaml) + YamlBindings.update_yml_source(temp_path, update_dict) comparison_bindings = YamlBindings()