diff --git a/README.md b/README.md index 991cfa6..6dcd38b 100644 --- a/README.md +++ b/README.md @@ -14,9 +14,9 @@ A set of CLI tools to manipulate YAML files (merge, delete, etc...) with comment (For development see section at the end) ``` $ pip install ruamel.yaml -$ export YAML_TOOLS_VERSION=0.2.0 +$ export YAML_TOOLS_VERSION=0.0.3 $ sudo wget https://raw.githubusercontent.com/thecodingmachine/yaml-tools/${YAML_TOOLS_VERSION}/src/yaml_tools.py -O /usr/local/bin/yaml-tools -$ sudo chmod +x /usr/bin/yaml-tools +$ sudo chmod +x /usr/bin/local/yaml-tools ``` ## Usage @@ -24,26 +24,36 @@ $ sudo chmod +x /usr/bin/yaml-tools $ yaml-tools [] ``` -There are only 2 commands at the moments : +There are 3 commands at the moments : ### merge -Merge two or more yaml files and preserve the comments +Merges two or more yaml files and preserves the comments. ``` -$ yaml-tools merge -i INPUTS [INPUTS ...] [-o OUTPUT] [--indent INDENT] +$ yaml-tools merge -i INPUTS [INPUTS ...] [-o OUTPUT] ``` - INPUTS: paths to input yaml files, which will be merged from the last to the first. - OUTPUT: path to output yaml file (or sys.stdout by default). -- INDENT: number of space(s) for each indent. ### delete -Delete one item from the input yaml file +Deletes one item/block (and its preceding comments) from the input yaml file. ``` -$ yaml-tools delete ITEM_PATH -i INPUT [-o OUTPUT] [--indent INDENT] +$ yaml-tools delete PATH_TO_KEY -i INPUT [-o OUTPUT] ``` -- ITEM_PATH: yaml item to be deleted, e.g. `key1.list[0].key2` +- PATH_TO_KEY: "path" to access the yaml item/block which will be deleted, e.g. `key1 0 key2` +- INPUT: path to input yaml file. +- OUTPUT: path to output yaml file (or sys.stdout by default). + +### comment (/!\ EXPERIMENTAL) +Comments one item/block from the input yaml file and preserves the comments. + +There are somme issues with comments which are at the end of any intermediate level/block, +and also commenting the last item from a list, so use it with caution. +``` +$ yaml-tools comment PATH_TO_KEY -i INPUT [-o OUTPUT] +``` +- PATH_TO_KEY: "path" to access the yaml item which will be commented, e.g. `key1 0 key2` - INPUT: path to input yaml file. - OUTPUT: path to output yaml file (or sys.stdout by default). -- INDENT: number of space(s) for each indent. ## Development @@ -53,9 +63,12 @@ $ yaml-tools delete ITEM_PATH -i INPUT [-o OUTPUT] [--indent INDENT] - Activate your venv with `.\venv\Scripts\activate` (Windows) or `source ./venv/bin/activate` (Linux or MacOS) - Install all required packages with `pip install -r requirements.txt` -## Running the tests +## Running tests ``` $ cd src/tests/ -$ python -m unittest discover + +$ python -m unittest discover +or +$ coverage run --rcfile=../../.coveragerc --source=.,.. -m unittest discover && coverage report -m ``` ## diff --git a/src/tests/__init__.py b/src/tests/__init__.py index 8b13789..e69de29 100644 --- a/src/tests/__init__.py +++ b/src/tests/__init__.py @@ -1 +0,0 @@ - diff --git a/src/tests/delete/expected_out.yml b/src/tests/delete/expected_out.yml index 27904f0..940b50c 100644 --- a/src/tests/delete/expected_out.yml +++ b/src/tests/delete/expected_out.yml @@ -2,8 +2,12 @@ test: foo: h: - - check: ok + # comment2 + - check: ok # comment3 ef: fefe - - ef: fefsegsegs + # comment4 + - ef: fefsegsegs # comment5 + - {} + # comment6 i: random bar: 1 diff --git a/src/tests/delete/file.yml b/src/tests/delete/file.yml index 3be377d..c807bbf 100644 --- a/src/tests/delete/file.yml +++ b/src/tests/delete/file.yml @@ -2,10 +2,14 @@ test: foo: h: - - check: ok + # comment2 + - check: ok # comment3 ef: fefe + # comment4 - check: not_ok - ef: fefsegsegs - - fqffqzfq + ef: fefsegsegs # comment5 + - check: not_ok # comment-nope + ef: nope # comment-nope + # comment6 i: random bar: 1 \ No newline at end of file diff --git a/src/tests/merge/expected_out.yml b/src/tests/merge/expected_out.yml index 99f76bd..69f1626 100644 --- a/src/tests/merge/expected_out.yml +++ b/src/tests/merge/expected_out.yml @@ -1,5 +1,39 @@ #comment1 test: - foo: 2 #comment1 + foo: + h: #comment1.0 + # comment2 + check: ok # comment3 + ef: fefe + # new-comment + check2: not_ok + ef2: fefsegsegs # comment5 + check3: not_ok # comment-ok + ef3: nope # comment-ok + # comment6 + # comment7 + i: # comment8 + toto: random bar: 3 #comment3 + foo2: + h: + # comment2 + - check: ok # comment3 + ef: fefe + # comment4 + - check: not_ok + ef: fefsegsegs + - check: not_ok + ef: nope # comment-ok + - check: okz # comment3 + ef: fefed + - check: not_okay + ef: fefsegdzasegs # comment5 + - check: not_ok # comment-ok + ef: nope # comment-ok + # comment6 + i: randomd foobar: 3 #comment3 + # foo + # end +# end 2 diff --git a/src/tests/merge/file1.yml b/src/tests/merge/file1.yml index 354e15a..005854e 100644 --- a/src/tests/merge/file1.yml +++ b/src/tests/merge/file1.yml @@ -1,4 +1,16 @@ -#comment1 test: - foo: 1 #comment1 - bar: 1 \ No newline at end of file + foo: + h: #ok + check: ook + ef: fefedfr + # new-comment + check2: not_ok_ + ef2: fefsegsegsf + check3: not_oke + ef3: nopee + i: + toto: random + # comment-bar + bar: 1 + # end nope +# end nope \ No newline at end of file diff --git a/src/tests/merge/file2.yml b/src/tests/merge/file2.yml index b4ccad0..edf0538 100644 --- a/src/tests/merge/file2.yml +++ b/src/tests/merge/file2.yml @@ -1,2 +1,31 @@ +#comment1 test: - foo: 2 \ No newline at end of file + foo: + h: #comment1.0 + # comment2 + check: ok # comment3 + ef: fefe + # comment4 + check2: not_ok + ef2: fefsegsegs # comment5 + check3: not_ok # comment-ok + ef3: nope # comment-ok + # comment6 + # comment7 + i: # comment8 + toto: random + bar: 1 + foo2: + h: + # comment2 + - check: ok # comment3 + ef: fefe + # comment4 + - check: not_ok + ef: fefsegsegs + - check: not_ok + ef: nope # comment-ok + i: random + # foo + # end +# end 2 \ No newline at end of file diff --git a/src/tests/merge/file3.yml b/src/tests/merge/file3.yml index 4c0ac28..dfbb1cd 100644 --- a/src/tests/merge/file3.yml +++ b/src/tests/merge/file3.yml @@ -1,3 +1,15 @@ test: bar: 3 #comment3 - foobar: 3 #comment3 \ No newline at end of file + foobar: 3 #comment3 + foo2: + h: + # comment2 + - check: okz # comment3 + ef: fefed + # comment4 + - check: not_okay + ef: fefsegdzasegs # comment5 + - check: not_ok # comment-ok + ef: nope # comment-ok + # comment6 + i: randomd \ No newline at end of file diff --git a/src/tests/tests.py b/src/tests/tests.py index 162eb04..ed36f53 100644 --- a/src/tests/tests.py +++ b/src/tests/tests.py @@ -1,9 +1,10 @@ +import sys import unittest -import ruamel.yaml + from ruamel.yaml import YAML +from ruamel.yaml import round_trip_load from ruamel.yaml.compat import StringIO -import sys sys.path.append('..') import yaml_tools @@ -23,9 +24,11 @@ class TestCommands(unittest.TestCase): merge_str_out = """ #comment1 test: + #ninja-comment2 foo: 2 #comment1 bar: 3 #comment3 foobar: 3 #comment3 + #ninja-comment3 """ def test_unrecognized_command(self): @@ -40,43 +43,48 @@ def test_fail_delete_command(self): sys.argv = ['yaml-tools', 'delete', 'unknownKey0', '-i', fi] self.assertRaises(KeyError, yaml_tools.main) - sys.argv = ['yaml-tools', 'delete', 'unknownKey1.foo', '-i', fi] - self.assertRaises(KeyError, yaml_tools.main) + sys.argv = ['yaml-tools', 'delete', 'unknownKey1', 'foo', '-i', fi] + self.assertRaises(RuntimeError, yaml_tools.main) - sys.argv = ['yaml-tools', 'delete', 'test.foo[0].check', '-i', fi] - self.assertRaises(TypeError, yaml_tools.main) + sys.argv = ['yaml-tools', 'delete', 'test', 'foo', '0', 'check', '-i', fi] + self.assertRaises(RuntimeError, yaml_tools.main) - sys.argv = ['yaml-tools', 'delete', 'test.foo.h[10].check', '-i', fi] - self.assertRaises(IndexError, yaml_tools.main) + sys.argv = ['yaml-tools', 'delete', 'test', 'foo', 'h', '10' 'check', '-i', fi] + self.assertRaises(RuntimeError, yaml_tools.main) - sys.argv = ['yaml-tools', 'delete', 'test.foo.h[1000]', '-i', fi] - self.assertRaises(IndexError, yaml_tools.main) + sys.argv = ['yaml-tools', 'delete', 'test', 'foo', 'h', '1000', '-i', fi] + self.assertRaises(RuntimeError, yaml_tools.main) - sys.argv = ['yaml-tools', 'delete', 'test.foo.unknownKey2[0]', '-i', fi] - self.assertRaises(KeyError, yaml_tools.main) + sys.argv = ['yaml-tools', 'delete', + 'test', 'foo', 'unknownKey2', '0', '-i', fi] + self.assertRaises(RuntimeError, yaml_tools.main) - sys.argv = ['yaml-tools', 'delete', 'test.foo.h[2].unknownKey3', '-i', fi] - self.assertRaises(TypeError, yaml_tools.main) + sys.argv = ['yaml-tools', 'delete', + 'test', 'foo', 'h', '2', 'unknownKey3', '-i', fi] + self.assertRaises(KeyError, yaml_tools.main) def test_3_str_merge_with_comment(self): str1 = """ #comment1 test: foo: 1 #comment1 + #ninja-comment1 bar: 1 """ str2 = """ test: + #ninja-comment2 foo: 2 """ str3 = """ test: bar: 3 #comment3 foobar: 3 #comment3 + #ninja-comment3 """ out = yaml_tools.successive_merge([str1, str2, str3]) - expected_out = ruamel.yaml.round_trip_load(self.merge_str_out) + expected_out = round_trip_load(self.merge_str_out) yml = MyYAML() out_str = yml.dump(out) @@ -104,9 +112,16 @@ def test_delete_item(self): fo = './delete/out.yml' feo = './delete/expected_out.yml' - sys.argv = ['yaml-tools', 'delete', 'test.foo.h[2]', '-i', fi, '-o', fo] + sys.argv = ['yaml-tools', 'delete', + 'test', 'foo', 'h', '2', 'check', '-i', fi, '-o', fo] yaml_tools.main() - sys.argv = ['yaml-tools', 'delete', 'test.foo.h[1].check', '-i', fo, '-o', fo] + + sys.argv = ['yaml-tools', 'delete', + 'test', 'foo', 'h', '2', 'ef', '-i', fo, '-o', fo] + yaml_tools.main() + + sys.argv = ['yaml-tools', 'delete', + 'test', 'foo', 'h', '1', 'check', '-i', fo, '-o', fo] yaml_tools.main() out_file = open(fo, 'r') @@ -116,6 +131,90 @@ def test_delete_item(self): expected_out_file.close() +class TestCommentCommand(unittest.TestCase): + def test_comment_commented_map_item(self): + str = """ +#comment1 +test: + #ninja-comment + foo: + sub-foo: 1 + bar: + sub-bar: 2 + baz: + sub-baz: 3 + """ + expected_str = """ +#comment1 +test: + #ninja-comment + foo: + sub-foo: 1 + #bar: + # sub-bar: 2 + baz: + sub-baz: 3 + """ + + data = round_trip_load(str, preserve_quotes=True) + path_to_key = ['test', 'bar'] + out = yaml_tools.comment_yaml_item(data, path_to_key, False) + expected_out = round_trip_load(expected_str, preserve_quotes=True) + + yml = MyYAML() + out_str = yml.dump(out) + expected_out_str = yml.dump(expected_out) + self.assertEqual(out_str, expected_out_str) + + def test_comment_commented_seq_item(self): + str = """ +#comment1 +test: +#ninja-comment +- foo: + sub-foo: 1 +- bar: + sub-bar: 2 +- baz: + sub-baz: 3 + """ + expected_str = """ +#comment1 +test: +#ninja-comment +- foo: + sub-foo: 1 +#- bar: +# sub-bar: 2 +- baz: + sub-baz: 3 + """ + + data = round_trip_load(str, preserve_quotes=True) + path_to_key = ['test', '1'] + out = yaml_tools.comment_yaml_item(data, path_to_key, True) + expected_out = round_trip_load(expected_str, preserve_quotes=True) + + yml = MyYAML() + out_str = yml.dump(out) + expected_out_str = yml.dump(expected_out) + self.assertEqual(out_str, expected_out_str) + + def test_fail_comment_commented(self): + str = """ +test: + foo: + sub-foo: 1 + bar: + - sub-bar: 1 + - sub-bar: 2 + """ + data = round_trip_load(str, preserve_quotes=True) + self.assertRaises(KeyError, yaml_tools.comment_yaml_item, data, ['unknown-key']) + self.assertRaises(RuntimeError, yaml_tools.comment_yaml_item, data, ['test', 'bar', '1000']) + self.assertRaises(RuntimeError, yaml_tools.comment_yaml_item, data, ['test', 'bar', 'NotAnInteger']) + + class TestMergeByType(unittest.TestCase): mock_scalar_1 = 'test: 1' mock_scalar_2 = 'test: 2' @@ -139,36 +238,40 @@ class TestMergeByType(unittest.TestCase): """ mock_None = '' - # from scalar to any + # from scalar to any def test_merge_scalar_to_scalar(self): - out = yaml_tools.successive_merge([self.mock_scalar_1, self.mock_scalar_2]) - expected_out = ruamel.yaml.round_trip_load(self.mock_scalar_2) + out = yaml_tools.successive_merge( + [self.mock_scalar_1, self.mock_scalar_2]) + expected_out = round_trip_load(self.mock_scalar_2) self.assertEqual(out, expected_out) def test_merge_scalar_to_dict(self): - self.assertRaises(TypeError, yaml_tools.successive_merge, [self.mock_dict_1, self.mock_scalar_2]) + self.assertRaises(TypeError, yaml_tools.successive_merge, [ + self.mock_dict_1, self.mock_scalar_2]) def test_merge_scalar_to_list(self): - out = yaml_tools.successive_merge([self.mock_list_1, self.mock_scalar_2]) + out = yaml_tools.successive_merge( + [self.mock_list_1, self.mock_scalar_2]) expected_out_str = """ test: - item1 - item2 - 2 """ - expected_out = ruamel.yaml.round_trip_load(expected_out_str) + expected_out = round_trip_load(expected_out_str) self.assertEqual(out, expected_out) - + def test_merge_scalar_to_None(self): out = yaml_tools.successive_merge([self.mock_None, self.mock_scalar_2]) - expected_out = ruamel.yaml.round_trip_load(self.mock_scalar_2) + expected_out = round_trip_load(self.mock_scalar_2) self.assertEqual(out, expected_out) # from dict to any def test_merge_dict_to_scalar(self): - self.assertRaises(TypeError, yaml_tools.successive_merge, [self.mock_scalar_1, self.mock_dict_2]) + self.assertRaises(TypeError, yaml_tools.successive_merge, [ + self.mock_scalar_1, self.mock_dict_2]) def test_merge_dict_to_dict(self): out = yaml_tools.successive_merge([self.mock_dict_1, self.mock_dict_2]) @@ -178,31 +281,34 @@ def test_merge_dict_to_dict(self): bar: 2 foobar: babar """ - expected_out = ruamel.yaml.round_trip_load(expected_out_str) + expected_out = round_trip_load(expected_out_str) self.assertEqual(out, expected_out) def test_merge_dict_to_list(self): - self.assertRaises(TypeError, yaml_tools.successive_merge, [self.mock_list_1, self.mock_dict_2]) - + self.assertRaises(TypeError, yaml_tools.successive_merge, [ + self.mock_list_1, self.mock_dict_2]) + def test_merge_dict_to_None(self): out = yaml_tools.successive_merge([self.mock_None, self.mock_dict_2]) - expected_out = ruamel.yaml.round_trip_load(self.mock_dict_2) + expected_out = round_trip_load(self.mock_dict_2) self.assertEqual(out, expected_out) - ## from list to any + # from list to any def test_merge_list_to_scalar(self): - out = yaml_tools.successive_merge([self.mock_scalar_1, self.mock_list_2]) + out = yaml_tools.successive_merge( + [self.mock_scalar_1, self.mock_list_2]) expected_out_str = """ test: - item3 - 1 """ # the scalar is appended at the end of the list - expected_out = ruamel.yaml.round_trip_load(expected_out_str) + expected_out = round_trip_load(expected_out_str) self.assertEqual(out, expected_out) def test_merge_list_to_dict(self): - self.assertRaises(TypeError, yaml_tools.successive_merge, [self.mock_dict_1, self.mock_list_2]) + self.assertRaises(TypeError, yaml_tools.successive_merge, [ + self.mock_dict_1, self.mock_list_2]) def test_merge_list_to_list(self): out = yaml_tools.successive_merge([self.mock_list_1, self.mock_list_2]) @@ -212,28 +318,32 @@ def test_merge_list_to_list(self): - item2 - item3 """ - expected_out = ruamel.yaml.round_trip_load(expected_out_str) + expected_out = round_trip_load(expected_out_str) self.assertEqual(out, expected_out) def test_merge_list_to_None(self): out = yaml_tools.successive_merge(['test: ', self.mock_list_2]) - expected_out = ruamel.yaml.round_trip_load(self.mock_list_2) + expected_out = round_trip_load(self.mock_list_2) self.assertEqual(out, expected_out) - ## from None to any + # from None to any def test_merge_None_to_any(self): out = yaml_tools.successive_merge([self.mock_None, self.mock_None]) - expected_out = ruamel.yaml.round_trip_load(self.mock_None) - self.assertEqual(out, expected_out, 'Merge None to None should succeed') + expected_out = round_trip_load(self.mock_None) + self.assertEqual(out, expected_out, + 'Merge None to None should succeed') out = yaml_tools.successive_merge([self.mock_scalar_1, self.mock_None]) - expected_out = ruamel.yaml.round_trip_load(self.mock_scalar_1) - self.assertEqual(out, expected_out, 'Merge None to scalar should succeed') + expected_out = round_trip_load(self.mock_scalar_1) + self.assertEqual(out, expected_out, + 'Merge None to scalar should succeed') out = yaml_tools.successive_merge([self.mock_dict_1, self.mock_None]) - expected_out = ruamel.yaml.round_trip_load(self.mock_dict_1) - self.assertEqual(out, expected_out, 'Merge None to dict should succeed') + expected_out = round_trip_load(self.mock_dict_1) + self.assertEqual(out, expected_out, + 'Merge None to dict should succeed') out = yaml_tools.successive_merge([self.mock_list_1, self.mock_None]) - expected_out = ruamel.yaml.round_trip_load(self.mock_list_1) - self.assertEqual(out, expected_out, 'Merge None to list should succeed') + expected_out = round_trip_load(self.mock_list_1) + self.assertEqual(out, expected_out, + 'Merge None to list should succeed') if __name__ == '__main__': # pragma: no cover diff --git a/src/yaml_tools.py b/src/yaml_tools.py index 457ddf0..0f92f9e 100755 --- a/src/yaml_tools.py +++ b/src/yaml_tools.py @@ -1,199 +1,346 @@ #!/usr/bin/env python -import sys import argparse -import ruamel.yaml -import re +import sys +from copy import deepcopy + +from ruamel.yaml import round_trip_dump, round_trip_load +from ruamel.yaml.comments import CommentedMap, CommentedSeq +from ruamel.yaml.error import StreamMark +from ruamel.yaml.tokens import CommentToken + + +## +# MERGE +# def get_type_error(dest, src, current_path): - return TypeError('Error trying to merge a {0} in a {1} at {2}'.format(type(src), type(dest), current_path)) + return TypeError('Error trying to merge a {0} in a {1} at ({2})'.format(type(src), type(dest), current_path)) + + +def copy_ca_comment_and_ca_end(dest, src): + # ruamel.yaml.Comment.ca contains 3 attributes : comment, items and end. We just copy comment and end here + if src.ca and dest.ca: + if src.ca.comment is not None: + if dest.ca.comment is None: + dest.ca.comment = [None, None] + if src.ca.comment[0] is not None: + dest.ca.comment[0] = src.ca.comment[0] + if src.ca.comment[1] is not None and len(src.ca.comment[1]) > 0: + dest.ca.comment[1] = src.ca.comment[1] + if len(src.ca.end) > 0: + dest.ca.end = src.ca.end -def _merge(dest, src, current_path=""): +def merge(dest, src, current_path=""): """ - (Recursively) merge a source object to an dest object (CommentedMap, CommentedSeq or other object) + (Recursively) merge a source object to an dest object (CommentedMap, CommentedSeq, scalar or None) and append the current position to current_path :return: the merged object """ - if isinstance(src, ruamel.yaml.comments.CommentedMap): - if isinstance(dest, ruamel.yaml.comments.CommentedMap): + if isinstance(src, CommentedMap): + if isinstance(dest, CommentedMap): for k in src: - dest[k] = _merge(dest[k], src[k], current_path + '.' + str(k)) if k in dest else src[k] - if k in src.ca._items and src.ca._items[k][2] and \ - src.ca._items[k][2].value.strip(): - dest.ca._items[k] = src.ca._items[k] # copy non-empty comment + dest[k] = merge(dest[k], src[k], current_path + '->' + str(k)) if k in dest else src[k] + if k in src.ca.items and src.ca.items[k][2] and src.ca.items[k][2].value.strip(): + # copy non empty 'items' comments + dest.ca.items[k] = src.ca.items[k] + copy_ca_comment_and_ca_end(dest, src) elif dest is None: return src else: raise get_type_error(dest, src, current_path) - elif isinstance(src, ruamel.yaml.comments.CommentedSeq): - if isinstance(dest, ruamel.yaml.comments.CommentedMap): - raise get_type_error(dest, src, current_path) - elif isinstance(dest, ruamel.yaml.comments.CommentedSeq): - dest.extend(src) + elif isinstance(src, CommentedSeq): + if isinstance(dest, CommentedSeq): + for i in src: + dest.append(i) + copy_ca_comment_and_ca_end(dest, src) + elif isinstance(dest, (str, int, float, bool)): + src.append(dest) + return src elif dest is None: return src else: - src.append(dest) + raise get_type_error(dest, src, current_path) + elif isinstance(src, (str, int, float, bool)): + if isinstance(dest, CommentedSeq): + dest.append(src) + elif isinstance(dest, (str, int, float, bool)): dest = src + else: + raise get_type_error(dest, src, current_path) elif src is None: return dest else: - if isinstance(dest, ruamel.yaml.comments.CommentedMap): - raise get_type_error(dest, src, current_path) - elif isinstance(dest, ruamel.yaml.comments.CommentedSeq): - dest.append(src) - else: - dest = src + raise get_type_error(dest, src, current_path) return dest def successive_merge(contents): """ - Successively merge a list of yaml contents by calling _merge() + Successively merge a list of yaml contents by calling merge() :param contents: list of yaml contents in str format :return: merged yaml in str format """ data = [] for i in contents: - data.append(ruamel.yaml.round_trip_load(i)) + data.append(round_trip_load(i, preserve_quotes=True)) for i in range(-1, -len(contents), -1): - final_data = _merge(data[i - 1], data[i], 'ROOT') + final_data = merge(data[i - 1], data[i], 'ROOT') return final_data -def has_valid_brackets(s): - """ - Check if the string s is in format "key[index]" - :param s: e.g. my_list[0] - :return: (True, key, index) or (None, None, None) - """ - list_regex = re.compile(r"\A\w+\[{1}\d+\]{1}\Z") - if list_regex.match(s) is not None: - key = s[:int(s.find('['))] - index = int(s[s.find('[') + 1: len(s) - 1]) - return True, key, index - return None, None, None +## +# DELETE and COMMENT +## -def get_dict_item(dic, item): - """ - Get one specific item from a dict - :param dic: the dict - :param item: a key or key[index], in string format - :return: the wanted item if found, otherwise raises an error - """ - is_array, key, index = has_valid_brackets(item) - if is_array: # if we want to access an item from an array - try: - got_item = dic[key][index] - except IndexError: - raise IndexError('list index out of range at "{}"'.format(item)) - except KeyError: - raise TypeError("'{}' is not a list".format(key)) +def str_or_int_map(s): + return int(s) if is_int(s) else s - else: # simple dict get - got_item = dic[item] - return got_item + +def is_int(s): + try: + int(s) + return True + except ValueError: + return False -def get_dict_item_from_path(dic, path): +def delete_yaml_item(data, path_to_key, data_contains_list=True): """ - Utility function to get one specific item from a dict given his "path" (in str format, e.g. "key1.list_key[0].key2") - :return: the item if found + Delete a yaml item given its path_to_key (e.g. [foo 0 bar]), and its direct previous comment(s) """ - if path == '': - return dic + if data_contains_list: + path_to_key = list(map(str_or_int_map, path_to_key)) - path_to_item = path.split('.') + parent = data.mlget(path_to_key[:-1], list_ok=data_contains_list) if len(path_to_key) > 1 else data + item_key = path_to_key[-1] - curr = dic - for p in path_to_item: - curr = get_dict_item(curr, p) - return curr + if isinstance(parent, CommentedMap): + if item_key not in parent: + raise KeyError("the key \'{}\' does not exist".format(item_key)) + preceding_comments = parent.ca.items.get(item_key, [None, None, None, None])[1] + del parent[item_key] + elif isinstance(parent, CommentedSeq): + if not is_int(item_key) or item_key >= len(parent): + raise RuntimeError("the key \'{}\' is not an integer or exceeds its parent's length".format(item_key)) + else: + preceding_comments = deepcopy(parent.ca.items.get(item_key, [None, None, None, None])[1]) + parent.pop(item_key) # CommentedSet.pop(idx) automatically shifts all ca.items' indexes ! + else: + raise RuntimeError("Couldn't reach the last item following the path_to_key " + str(path_to_key)) + + return data, preceding_comments -def delete(): +def comment_yaml_item(data, path_to_key, data_contains_list=True): """ - Sub-command, see main() + (EXPERIMENTAL) Comment a yaml item given its path_to_key (e.g. [foo 0 bar]), with comment preservation + Inspired from https://stackoverflow.com/a/43927974 @cherrot """ - parser = argparse.ArgumentParser(description='Delete one item from the input yaml file') - parser.add_argument('item_path', type=str, help=' Yaml item to be deleted, e.g. "key1.list[0].key2"') - parser.add_argument('-i', '--input', type=str, help=' Path to the input yaml files', required=True) - parser.add_argument('-o', '--output', type=str, help='Path to the output file, or stdout by default') - parser.add_argument('--indent', type=int, help='Number of space(s) for each indent', default=2) + if data_contains_list: + path_to_key = list(map(str_or_int_map, path_to_key)) - args = parser.parse_args(sys.argv[2:]) - input_file = open(args.input, 'r') - data = ruamel.yaml.round_trip_load(input_file.read()) - input_file.close() + parent = data.mlget(path_to_key[:-1], list_ok=data_contains_list) if len(path_to_key) > 1 else data + item_key = path_to_key[-1] + deleted_item = item_key - path_list = args.item_path.split('.') - item_parent = get_dict_item_from_path(data, '.'.join(path_list[:-1])) + next_key = None - item_to_delete = path_list[-1] - is_array, key, index = has_valid_brackets(item_to_delete) - try: - if is_array: - item_parent[key][index] # to trigger a KeyError if not found - del item_parent[key][index] + if isinstance(parent, CommentedMap): + if item_key not in parent: + raise KeyError("the key \'{}\' does not exist".format(item_key)) + # don't just pop the value for item_key that way you lose comments + # in the original YAML, instead deepcopy and delete what is not needed + block_copy = deepcopy(parent) + found = False + keys = [k for k in parent.keys()] + for key in reversed(keys): + if key == item_key: + found = True + else: + if not found: + next_key = key + del block_copy[key] + + # now delete the key and its value, but preserve its preceding comments + preceding_comments = parent.ca.items.get(item_key, [None, None, None, None])[1] + + if next_key is None: + if parent.ca.comment is None: + parent.ca.comment = [None, []] + if parent.ca.comment[1] is None: + parent.ca.comment[1] = [] + comment_list = parent.ca.comment[1] + else: + comment_list = parent.ca.items.get(next_key, [None, None, None, None])[1] + if comment_list is None: + parent.ca.items[next_key] = [None, [], None, None] + comment_list = parent.ca.items.get(next_key)[1] + if preceding_comments is not None: + for c in reversed(preceding_comments): + comment_list.insert(0, c) + del parent[item_key] + elif isinstance(parent, CommentedSeq): + if not is_int(item_key) or item_key >= len(parent): + raise RuntimeError("the key \'{}\' is not an integer or exceeds its parent's length".format(item_key)) else: - item_parent[item_to_delete] - del item_parent[item_to_delete] - except (AttributeError, KeyError, IndexError, TypeError): - print("An error occurred when deleting '{}' :".format(item_to_delete)) - raise + block_copy = deepcopy(parent) + for i in reversed(range(len(parent))): + if i != item_key: + del block_copy[i] - output_file = open(args.output, 'w') if args.output else sys.stdout - ruamel.yaml.round_trip_dump(data, output_file, indent=args.indent) - output_file.close() + next_key = item_key + preceding_comments = deepcopy(parent.ca.items.get(item_key, [None, None, None, None])[1]) + parent.pop(item_key) # CommentedSet.pop(idx) automatically shifts all ca.items' indexes ! + if len(parent) == 1 or next_key == len(parent): + comment_list = parent.ca.end # TODO: fix this, the appended comments don't show up in some case + else: + comment_list = parent.ca.items.get(next_key, [None, None, None, None])[1] + if comment_list is None: + parent.ca.items[next_key] = [None, [], None, None] + comment_list = parent.ca.items.get(next_key)[1] -def merge(): - """ - Sub-command, see main() - """ - parser = argparse.ArgumentParser(description='Merge two or more yaml files and preserve the comments') - parser.add_argument('-i', '--inputs', nargs='+', type=str, help=' List of input yaml files', - required=True) - parser.add_argument('-o', '--output', type=str, help='Path to the output file, or stdout by default') - parser.add_argument('--indent', type=int, help='Number of space(s) for each indent', default=2) + if preceding_comments is not None: + for c in reversed(preceding_comments): + comment_list.insert(0, c) + else: + raise RuntimeError("Couldn't reach the last item following the path_to_key " + str(path_to_key)) - args = parser.parse_args(sys.argv[2:]) + key_dept = len(path_to_key) - 1 + if is_int(path_to_key[-1]) and key_dept > 0: + key_dept = key_dept - 1 + comment_list_copy = deepcopy(comment_list) + del comment_list[:] - file_contents = [] - for f in args.inputs: - file = open(f, 'r') - file_contents.append(file.read()) - file.close() + start_mark = StreamMark(None, None, None, 2 * key_dept) + skip = True + for line in round_trip_dump(block_copy).splitlines(True): + if skip: + if line.strip(' ').startswith('#'): # and deleted_item not in line: + continue + skip = False + comment_list.append(CommentToken('#' + line, start_mark, None)) + comment_list.extend(comment_list_copy) + + return data - out_content = successive_merge(file_contents) - output_file = open(args.output, 'w') if args.output else sys.stdout - ruamel.yaml.round_trip_dump(out_content, output_file, indent=args.indent) - output_file.close() + +### +# main and commands +### def main(): parser = argparse.ArgumentParser( - description='A set of CLI tools to manipulate YAML files (merge, delete, etc...) with comment preservation', + description='A set of CLI tools to manipulate YAML files (merge, delete, comment, etc...) \ + with comment preservation', usage='''yaml-tools [] -At the moment there are only two commands available: +At the moment there are three commands available: merge Merge two or more yaml files and preserve the comments - delete Delete one item from the input yaml file''') + delete Delete an item (and all its child items) given its path from the input yaml file + comment Comment an item (and all its child items) given its path from the input yaml file''') parser.add_argument('command', help='Sub-command to run') # parse_args defaults to [1:] for args, but you need to # exclude the rest of the args too, or validation will fail args = parser.parse_args(sys.argv[1:2]) if args.command == 'merge': - merge() + merge_command() elif args.command == 'delete': - delete() + delete_command() + elif args.command == 'comment': + comment_command() else: print('Unrecognized command') parser.print_help() exit(1) +def merge_command(): + """ + Sub-command, see main() + """ + parser = argparse.ArgumentParser( + description='Merge two or more yaml files and preserve the comments') + parser.add_argument('-i', '--inputs', nargs='+', type=str, + help=' List of input yaml files, merged from the last to the first', + required=True) + parser.add_argument('-o', '--output', type=str, + help='Path to the output file, or stdout by default') + parser.add_argument('--indent', type=int, + help='Number of space(s) for each indent', default=2) + + args = parser.parse_args(sys.argv[2:]) + + file_contents = [] + for f in args.inputs: + file = open(f, 'r') + file_contents.append(file.read()) + file.close() + + out_content = successive_merge(file_contents) + output_file = open(args.output, 'w') if args.output else sys.stdout + round_trip_dump(out_content, output_file) + output_file.close() + + +def delete_command(): + """ + Sub-command, see main() + """ + parser = argparse.ArgumentParser( + description='Delete one item from the input yaml file') + parser.add_argument('path_to_key', type=str, nargs='+', + help=' Yaml item to be deleted, e.g. "foo 0 bar"') + parser.add_argument('-i', '--input', type=str, + help=' Path to the input yaml files', required=True) + parser.add_argument('-o', '--output', type=str, + help='Path to the output file, or stdout by default') + parser.add_argument('--indent', type=int, + help='Number of space(s) for each indent', default=2) + + args = parser.parse_args(sys.argv[2:]) + input_file = open(args.input, 'r') + data = round_trip_load(input_file.read(), preserve_quotes=True) + input_file.close() + + output_data, _ = delete_yaml_item(data, args.path_to_key, True) + + output_file = open(args.output, 'w') if args.output else sys.stdout + round_trip_dump(output_data, output_file) + output_file.close() + + +def comment_command(): # pragma: no cover + """ + Sub-command, see main() + """ + # TODO: refactor this command with delete ? + parser = argparse.ArgumentParser( + description='Comment one item from the input yaml file') + parser.add_argument('path_to_key', type=str, nargs='+', + help=' Yaml item to be commented, e.g. "foo 0 bar"') + parser.add_argument('-i', '--input', type=str, + help=' Path to the input yaml file', required=True) + parser.add_argument('-o', '--output', type=str, + help='Path to the output file, or stdout by default') + parser.add_argument('--indent', type=int, + help='Number of space(s) for each indent', default=2) + + args = parser.parse_args(sys.argv[2:]) + input_file = open(args.input, 'r') + data = round_trip_load(input_file.read(), preserve_quotes=True) + input_file.close() + + output_data = comment_yaml_item(data, args.path_to_key, True) + + output_file = open(args.output, 'w') if args.output else sys.stdout + round_trip_dump(output_data, output_file) + output_file.close() + + if __name__ == '__main__': # pragma: no cover main()