From 72e6b64aef25c1e58147130f1a6f98bae6f3a1d7 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Thu, 14 May 2020 16:01:45 -0600 Subject: [PATCH 1/9] add an exclude option --- manic/checkout.py | 5 ++++- manic/externals_description.py | 20 ++++++++++++-------- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/manic/checkout.py b/manic/checkout.py index edc5655954..793002d58b 100755 --- a/manic/checkout.py +++ b/manic/checkout.py @@ -279,6 +279,9 @@ def commandline_arguments(args=None): help='The externals description filename. ' 'Default: %(default)s.') + parser.add_argument('-x', '--exclude', nargs='*', + help='Component(s) listed in the externals file which should be ignored.' ) + parser.add_argument('-o', '--optional', action='store_true', default=False, help='By default only the required externals ' 'are checked out. This flag will also checkout the ' @@ -362,7 +365,7 @@ def main(args): root_dir = os.path.abspath(os.getcwd()) external_data = read_externals_description_file(root_dir, args.externals) external = create_externals_description( - external_data, components=args.components) + external_data, components=args.components, exclude=args.exclude) for comp in args.components: if comp not in external.keys(): diff --git a/manic/externals_description.py b/manic/externals_description.py index bd644c9814..4c0d6971be 100644 --- a/manic/externals_description.py +++ b/manic/externals_description.py @@ -264,18 +264,18 @@ def read_gitmodules_file(root_dir, file_name): return externals_description def create_externals_description( - model_data, model_format='cfg', components=None, parent_repo=None): + model_data, model_format='cfg', components=None, exclude=None, parent_repo=None): """Create the a externals description object from the provided data """ externals_description = None if model_format == 'dict': externals_description = ExternalsDescriptionDict( - model_data, components=components) + model_data, components=components, exclude=exclude) elif model_format == 'cfg': major, _, _ = get_cfg_schema_version(model_data) if major == 1: externals_description = ExternalsDescriptionConfigV1( - model_data, components=components, parent_repo=parent_repo) + model_data, components=components, exclude=exclude, parent_repo=parent_repo) else: msg = ('Externals description file has unsupported schema ' 'version "{0}".'.format(major)) @@ -710,7 +710,7 @@ class ExternalsDescriptionDict(ExternalsDescription): """ - def __init__(self, model_data, components=None): + def __init__(self, model_data, components=None, exclude=None): """Parse a native dictionary into a externals description. """ ExternalsDescription.__init__(self) @@ -725,6 +725,10 @@ def __init__(self, model_data, components=None): for key in model_data.items(): if key not in components: del model_data[key] + if exclude: + for key in model_data.items(): + if key in exclude: + del model_data[key] self.update(model_data) self._check_user_input() @@ -736,7 +740,7 @@ class ExternalsDescriptionConfigV1(ExternalsDescription): """ - def __init__(self, model_data, components=None, parent_repo=None): + def __init__(self, model_data, components=None, exclude=None, parent_repo=None): """Convert the config data into a standardized dict that can be used to construct the source objects @@ -749,7 +753,7 @@ def __init__(self, model_data, components=None, parent_repo=None): get_cfg_schema_version(model_data) self._verify_schema_version() self._remove_metadata(model_data) - self._parse_cfg(model_data, components=components) + self._parse_cfg(model_data, components=components, exclude=exclude) self._check_user_input() @staticmethod @@ -761,7 +765,7 @@ def _remove_metadata(model_data): """ model_data.remove_section(DESCRIPTION_SECTION) - def _parse_cfg(self, cfg_data, components=None): + def _parse_cfg(self, cfg_data, components=None, exclude=None): """Parse a config_parser object into a externals description. """ def list_to_dict(input_list, convert_to_lower_case=True): @@ -778,7 +782,7 @@ def list_to_dict(input_list, convert_to_lower_case=True): for section in cfg_data.sections(): name = config_string_cleaner(section.lower().strip()) - if components and name not in components: + if (components and name not in components) or (exclude and name in exclude): continue self[name] = {} self[name].update(list_to_dict(cfg_data.items(section))) From 21affe33ccb6567963eddbb97c9a536f2604c3e5 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Fri, 15 May 2020 08:38:27 -0600 Subject: [PATCH 2/9] fix formatting issue --- manic/checkout.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manic/checkout.py b/manic/checkout.py index 793002d58b..c07736450c 100755 --- a/manic/checkout.py +++ b/manic/checkout.py @@ -280,7 +280,7 @@ def commandline_arguments(args=None): 'Default: %(default)s.') parser.add_argument('-x', '--exclude', nargs='*', - help='Component(s) listed in the externals file which should be ignored.' ) + help='Component(s) listed in the externals file which should be ignored.') parser.add_argument('-o', '--optional', action='store_true', default=False, help='By default only the required externals ' From d43710864f29d697f56ff5b61f5f181f583e1ab8 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Fri, 15 May 2020 08:58:09 -0600 Subject: [PATCH 3/9] add a test --- test/test_sys_checkout.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/test/test_sys_checkout.py b/test/test_sys_checkout.py index df726f2b70..da0a6dcb68 100644 --- a/test/test_sys_checkout.py +++ b/test/test_sys_checkout.py @@ -1347,6 +1347,23 @@ def test_container_component(self): self.status_args) self._check_container_component_post_checkout2(overall, tree) + def test_container_exclude_component(self): + """Verify that optional component checkout works + """ + # create the test repository + under_test_dir = self.setup_test_repo(CONTAINER_REPO_NAME) + + # create the top level externals file + self._generator.container_full(under_test_dir) + + # inital checkout, first try a nonexistant component argument noref + checkout_args = ['--exclude simp_opt'] + checkout_args.extend(self.checkout_args) + + overall, tree = self.execute_cmd_in_dir(under_test_dir, checkout_args) + + self._check_container_component_post_checkout2(overall, tree) + def test_mixed_simple(self): """Verify that a mixed use repo can serve as a 'full' container, pulling in a set of externals and a seperate set of sub-externals. From a580a570b35f20a295f5c7e5b1fe09fdf6b73f20 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Fri, 15 May 2020 09:39:49 -0600 Subject: [PATCH 4/9] push test --- test/test_sys_checkout.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/test/test_sys_checkout.py b/test/test_sys_checkout.py index da0a6dcb68..532445281a 100644 --- a/test/test_sys_checkout.py +++ b/test/test_sys_checkout.py @@ -819,10 +819,15 @@ def _check_container_component_post_checkout(self, overall, tree): def _check_container_component_post_checkout2(self, overall, tree): self.assertEqual(overall, 0) - self._check_simple_opt_ok(tree) self._check_simple_tag_empty(tree) self._check_simple_branch_ok(tree) + def _check_container_component_post_checkout3(self, overall, tree): + self.assertEqual(overall, 0) + self.assertFalse("simp_opt" in tree) + self._check_simple_tag_ok(tree) + self._check_simple_branch_ok(tree) + def _check_container_full_post_checkout(self, overall, tree): self.assertEqual(overall, 0) self._check_simple_tag_ok(tree) @@ -1348,7 +1353,7 @@ def test_container_component(self): self._check_container_component_post_checkout2(overall, tree) def test_container_exclude_component(self): - """Verify that optional component checkout works + """Verify that exclude component checkout works """ # create the test repository under_test_dir = self.setup_test_repo(CONTAINER_REPO_NAME) @@ -1357,12 +1362,13 @@ def test_container_exclude_component(self): self._generator.container_full(under_test_dir) # inital checkout, first try a nonexistant component argument noref - checkout_args = ['--exclude simp_opt'] + checkout_args = ['--exclude','simp_opt'] checkout_args.extend(self.checkout_args) - + save_checkout_args = self.checkout_args + self.checkout_args = checkout_args overall, tree = self.execute_cmd_in_dir(under_test_dir, checkout_args) - - self._check_container_component_post_checkout2(overall, tree) + self._check_container_component_post_checkout3(overall, tree) + self.checkout_args = save_checkout_args def test_mixed_simple(self): """Verify that a mixed use repo can serve as a 'full' container, From be85b7d1bd32363ddf31cca7470c68a716add1a8 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Mon, 18 May 2020 12:19:20 -0600 Subject: [PATCH 5/9] fix the test --- test/test_sys_checkout.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/test/test_sys_checkout.py b/test/test_sys_checkout.py index 532445281a..2989a92a78 100644 --- a/test/test_sys_checkout.py +++ b/test/test_sys_checkout.py @@ -1361,14 +1361,13 @@ def test_container_exclude_component(self): # create the top level externals file self._generator.container_full(under_test_dir) - # inital checkout, first try a nonexistant component argument noref - checkout_args = ['--exclude','simp_opt'] + # inital checkout, exclude simp_opt + checkout_args = ['--exclude','simp_opt', '--logging'] checkout_args.extend(self.checkout_args) - save_checkout_args = self.checkout_args - self.checkout_args = checkout_args + overall, tree = self.execute_cmd_in_dir(under_test_dir, checkout_args) + checkout_args.append("--status") overall, tree = self.execute_cmd_in_dir(under_test_dir, checkout_args) self._check_container_component_post_checkout3(overall, tree) - self.checkout_args = save_checkout_args def test_mixed_simple(self): """Verify that a mixed use repo can serve as a 'full' container, From afab352c861248e0b9390bb75f02c19cb12c9477 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Mon, 18 May 2020 12:25:46 -0600 Subject: [PATCH 6/9] fix lint issue --- test/test_sys_checkout.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_sys_checkout.py b/test/test_sys_checkout.py index 2989a92a78..007f5e4897 100644 --- a/test/test_sys_checkout.py +++ b/test/test_sys_checkout.py @@ -1362,7 +1362,7 @@ def test_container_exclude_component(self): self._generator.container_full(under_test_dir) # inital checkout, exclude simp_opt - checkout_args = ['--exclude','simp_opt', '--logging'] + checkout_args = ['--exclude', 'simp_opt'] checkout_args.extend(self.checkout_args) overall, tree = self.execute_cmd_in_dir(under_test_dir, checkout_args) checkout_args.append("--status") From 418173ffdaf9f4e5602cc1f5df3ae67bc9345827 Mon Sep 17 00:00:00 2001 From: Steve Goldhaber Date: Mon, 18 May 2020 22:34:13 -0600 Subject: [PATCH 7/9] Added tests for ExternalsDescriptionDict --- .gitignore | 3 + test/test_unit_externals_description.py | 76 +++++++++++++++++++++++++ 2 files changed, 79 insertions(+) diff --git a/.gitignore b/.gitignore index 411de5d96e..a71ac0cd75 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,6 @@ components/ # generated python files *.pyc + +# test tmp file +test/tmp diff --git a/test/test_unit_externals_description.py b/test/test_unit_externals_description.py index 637f760ee5..1639f7d3b2 100644 --- a/test/test_unit_externals_description.py +++ b/test/test_unit_externals_description.py @@ -342,6 +342,39 @@ def setup_config(self): # NOTE(goldy, 2019-03) Should test other possible keywords such as # fetchRecurseSubmodules, ignore, and shallow + def setup_dict_config(self): + """Create the full container dictionary with simple and mixed use + externals + + """ + rdatat = {ExternalsDescription.PROTOCOL: 'git', + ExternalsDescription.REPO_URL: 'simple-ext.git', + ExternalsDescription.TAG: 'tag1'} + rdatab = {ExternalsDescription.PROTOCOL: 'git', + ExternalsDescription.REPO_URL: 'simple-ext.git', + ExternalsDescription.BRANCH: 'feature2'} + rdatam = {ExternalsDescription.PROTOCOL: 'git', + ExternalsDescription.REPO_URL: 'mixed-cont-ext.git', + ExternalsDescription.BRANCH: 'master'} + desc = {'simp_tag': {ExternalsDescription.REQUIRED: True, + ExternalsDescription.PATH: 'simp_tag', + ExternalsDescription.EXTERNALS: EMPTY_STR, + ExternalsDescription.REPO: rdatat}, + 'simp_branch' : {ExternalsDescription.REQUIRED: True, + ExternalsDescription.PATH: 'simp_branch', + ExternalsDescription.EXTERNALS: EMPTY_STR, + ExternalsDescription.REPO: rdatab}, + 'simp_opt': {ExternalsDescription.REQUIRED: False, + ExternalsDescription.PATH: 'simp_opt', + ExternalsDescription.EXTERNALS: EMPTY_STR, + ExternalsDescription.REPO: rdatat}, + 'mixed_req': {ExternalsDescription.REQUIRED: True, + ExternalsDescription.PATH: 'mixed_req', + ExternalsDescription.EXTERNALS: 'sub-ext.cfg', + ExternalsDescription.REPO: rdatam}} + + return desc + def test_cfg_v1_ok(self): """Test that a correct cfg v1 object is created by create_externals_description @@ -379,6 +412,49 @@ def test_dict(self): ext = create_externals_description(desc, model_format='dict') self.assertIsInstance(ext, ExternalsDescriptionDict) + def test_container_component_dict(self): + """Verify that create_externals_description works with a dictionary + """ + # create the top level externals file + desc = self.setup_dict_config() + # Check external with all repos + external = create_externals_description(desc, model_format='dict') + self.assertIsInstance(external, ExternalsDescriptionDict) + self.assertTrue('simp_tag' in external) + self.assertTrue('simp_branch' in external) + self.assertTrue('simp_opt' in external) + self.assertTrue('mixed_req' in external) + + def test_container_exclude_component_dict(self): + """Verify that exclude component checkout works with a dictionary + """ + # create the top level externals file + desc = self.setup_dict_config() + # Test an excluded repo + external = create_externals_description(desc, model_format='dict', + exclude=['simp_tag', + 'simp_opt']) + self.assertIsInstance(external, ExternalsDescriptionDict) + self.assertFalse('simp_tag' in external) + self.assertTrue('simp_branch' in external) + self.assertFalse('simp_opt' in external) + self.assertTrue('mixed_req' in external) + + def test_container_opt_component_dict(self): + """Verify that exclude component checkout works with a dictionary + """ + # create the top level externals file + desc = self.setup_dict_config() + # Test an excluded repo + external = create_externals_description(desc, model_format='dict', + components=['simp_tag', + 'simp_opt']) + self.assertIsInstance(external, ExternalsDescriptionDict) + self.assertTrue('simp_tag' in external) + self.assertFalse('simp_branch' in external) + self.assertTrue('simp_opt' in external) + self.assertFalse('mixed_req' in external) + def test_cfg_unknown_version(self): """Test that a runtime error is raised when an unknown file version is received From 49cd5e890a4d417c387cbb58d6dbb1f8188880b6 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Tue, 19 May 2020 08:23:30 -0600 Subject: [PATCH 8/9] fix lint issues --- manic/externals_description.py | 5 +++-- test/test_unit_externals_description.py | 6 +++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/manic/externals_description.py b/manic/externals_description.py index 4c0d6971be..498d8d11ce 100644 --- a/manic/externals_description.py +++ b/manic/externals_description.py @@ -722,11 +722,12 @@ def __init__(self, model_data, components=None, exclude=None): self._input_patch = 0 self._verify_schema_version() if components: - for key in model_data.items(): + for key in list(model_data.keys()): if key not in components: del model_data[key] + if exclude: - for key in model_data.items(): + for key in list(model_data.keys()): if key in exclude: del model_data[key] diff --git a/test/test_unit_externals_description.py b/test/test_unit_externals_description.py index 1639f7d3b2..8e3dffac43 100644 --- a/test/test_unit_externals_description.py +++ b/test/test_unit_externals_description.py @@ -412,7 +412,7 @@ def test_dict(self): ext = create_externals_description(desc, model_format='dict') self.assertIsInstance(ext, ExternalsDescriptionDict) - def test_container_component_dict(self): + def test_cfg_component_dict(self): """Verify that create_externals_description works with a dictionary """ # create the top level externals file @@ -425,7 +425,7 @@ def test_container_component_dict(self): self.assertTrue('simp_opt' in external) self.assertTrue('mixed_req' in external) - def test_container_exclude_component_dict(self): + def test_cfg_exclude_component_dict(self): """Verify that exclude component checkout works with a dictionary """ # create the top level externals file @@ -440,7 +440,7 @@ def test_container_exclude_component_dict(self): self.assertFalse('simp_opt' in external) self.assertTrue('mixed_req' in external) - def test_container_opt_component_dict(self): + def test_cfg_opt_component_dict(self): """Verify that exclude component checkout works with a dictionary """ # create the top level externals file From 2aa014a1b577780c75b5d8bfac67c09b9066dc27 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Tue, 19 May 2020 10:02:17 -0600 Subject: [PATCH 9/9] fix lint issue --- test/test_unit_externals_description.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/test_unit_externals_description.py b/test/test_unit_externals_description.py index 8e3dffac43..0b1248f670 100644 --- a/test/test_unit_externals_description.py +++ b/test/test_unit_externals_description.py @@ -342,7 +342,8 @@ def setup_config(self): # NOTE(goldy, 2019-03) Should test other possible keywords such as # fetchRecurseSubmodules, ignore, and shallow - def setup_dict_config(self): + @staticmethod + def setup_dict_config(): """Create the full container dictionary with simple and mixed use externals