diff --git a/.gitignore b/.gitignore index 411de5d96e..a71ac0cd75 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,6 @@ components/ # generated python files *.pyc + +# test tmp file +test/tmp diff --git a/manic/checkout.py b/manic/checkout.py index edc5655954..c07736450c 100755 --- a/manic/checkout.py +++ b/manic/checkout.py @@ -279,6 +279,9 @@ def commandline_arguments(args=None): help='The externals description filename. ' 'Default: %(default)s.') + parser.add_argument('-x', '--exclude', nargs='*', + help='Component(s) listed in the externals file which should be ignored.') + parser.add_argument('-o', '--optional', action='store_true', default=False, help='By default only the required externals ' 'are checked out. This flag will also checkout the ' @@ -362,7 +365,7 @@ def main(args): root_dir = os.path.abspath(os.getcwd()) external_data = read_externals_description_file(root_dir, args.externals) external = create_externals_description( - external_data, components=args.components) + external_data, components=args.components, exclude=args.exclude) for comp in args.components: if comp not in external.keys(): diff --git a/manic/externals_description.py b/manic/externals_description.py index bd644c9814..498d8d11ce 100644 --- a/manic/externals_description.py +++ b/manic/externals_description.py @@ -264,18 +264,18 @@ def read_gitmodules_file(root_dir, file_name): return externals_description def create_externals_description( - model_data, model_format='cfg', components=None, parent_repo=None): + model_data, model_format='cfg', components=None, exclude=None, parent_repo=None): """Create the a externals description object from the provided data """ externals_description = None if model_format == 'dict': externals_description = ExternalsDescriptionDict( - model_data, components=components) + model_data, components=components, exclude=exclude) elif model_format == 'cfg': major, _, _ = get_cfg_schema_version(model_data) if major == 1: externals_description = ExternalsDescriptionConfigV1( - model_data, components=components, parent_repo=parent_repo) + model_data, components=components, exclude=exclude, parent_repo=parent_repo) else: msg = ('Externals description file has unsupported schema ' 'version "{0}".'.format(major)) @@ -710,7 +710,7 @@ class ExternalsDescriptionDict(ExternalsDescription): """ - def __init__(self, model_data, components=None): + def __init__(self, model_data, components=None, exclude=None): """Parse a native dictionary into a externals description. """ ExternalsDescription.__init__(self) @@ -722,10 +722,15 @@ def __init__(self, model_data, components=None): self._input_patch = 0 self._verify_schema_version() if components: - for key in model_data.items(): + for key in list(model_data.keys()): if key not in components: del model_data[key] + if exclude: + for key in list(model_data.keys()): + if key in exclude: + del model_data[key] + self.update(model_data) self._check_user_input() @@ -736,7 +741,7 @@ class ExternalsDescriptionConfigV1(ExternalsDescription): """ - def __init__(self, model_data, components=None, parent_repo=None): + def __init__(self, model_data, components=None, exclude=None, parent_repo=None): """Convert the config data into a standardized dict that can be used to construct the source objects @@ -749,7 +754,7 @@ def __init__(self, model_data, components=None, parent_repo=None): get_cfg_schema_version(model_data) self._verify_schema_version() self._remove_metadata(model_data) - self._parse_cfg(model_data, components=components) + self._parse_cfg(model_data, components=components, exclude=exclude) self._check_user_input() @staticmethod @@ -761,7 +766,7 @@ def _remove_metadata(model_data): """ model_data.remove_section(DESCRIPTION_SECTION) - def _parse_cfg(self, cfg_data, components=None): + def _parse_cfg(self, cfg_data, components=None, exclude=None): """Parse a config_parser object into a externals description. """ def list_to_dict(input_list, convert_to_lower_case=True): @@ -778,7 +783,7 @@ def list_to_dict(input_list, convert_to_lower_case=True): for section in cfg_data.sections(): name = config_string_cleaner(section.lower().strip()) - if components and name not in components: + if (components and name not in components) or (exclude and name in exclude): continue self[name] = {} self[name].update(list_to_dict(cfg_data.items(section))) diff --git a/test/test_sys_checkout.py b/test/test_sys_checkout.py index df726f2b70..007f5e4897 100644 --- a/test/test_sys_checkout.py +++ b/test/test_sys_checkout.py @@ -819,10 +819,15 @@ def _check_container_component_post_checkout(self, overall, tree): def _check_container_component_post_checkout2(self, overall, tree): self.assertEqual(overall, 0) - self._check_simple_opt_ok(tree) self._check_simple_tag_empty(tree) self._check_simple_branch_ok(tree) + def _check_container_component_post_checkout3(self, overall, tree): + self.assertEqual(overall, 0) + self.assertFalse("simp_opt" in tree) + self._check_simple_tag_ok(tree) + self._check_simple_branch_ok(tree) + def _check_container_full_post_checkout(self, overall, tree): self.assertEqual(overall, 0) self._check_simple_tag_ok(tree) @@ -1347,6 +1352,23 @@ def test_container_component(self): self.status_args) self._check_container_component_post_checkout2(overall, tree) + def test_container_exclude_component(self): + """Verify that exclude component checkout works + """ + # create the test repository + under_test_dir = self.setup_test_repo(CONTAINER_REPO_NAME) + + # create the top level externals file + self._generator.container_full(under_test_dir) + + # inital checkout, exclude simp_opt + checkout_args = ['--exclude', 'simp_opt'] + checkout_args.extend(self.checkout_args) + overall, tree = self.execute_cmd_in_dir(under_test_dir, checkout_args) + checkout_args.append("--status") + overall, tree = self.execute_cmd_in_dir(under_test_dir, checkout_args) + self._check_container_component_post_checkout3(overall, tree) + def test_mixed_simple(self): """Verify that a mixed use repo can serve as a 'full' container, pulling in a set of externals and a seperate set of sub-externals. diff --git a/test/test_unit_externals_description.py b/test/test_unit_externals_description.py index 637f760ee5..0b1248f670 100644 --- a/test/test_unit_externals_description.py +++ b/test/test_unit_externals_description.py @@ -342,6 +342,40 @@ def setup_config(self): # NOTE(goldy, 2019-03) Should test other possible keywords such as # fetchRecurseSubmodules, ignore, and shallow + @staticmethod + def setup_dict_config(): + """Create the full container dictionary with simple and mixed use + externals + + """ + rdatat = {ExternalsDescription.PROTOCOL: 'git', + ExternalsDescription.REPO_URL: 'simple-ext.git', + ExternalsDescription.TAG: 'tag1'} + rdatab = {ExternalsDescription.PROTOCOL: 'git', + ExternalsDescription.REPO_URL: 'simple-ext.git', + ExternalsDescription.BRANCH: 'feature2'} + rdatam = {ExternalsDescription.PROTOCOL: 'git', + ExternalsDescription.REPO_URL: 'mixed-cont-ext.git', + ExternalsDescription.BRANCH: 'master'} + desc = {'simp_tag': {ExternalsDescription.REQUIRED: True, + ExternalsDescription.PATH: 'simp_tag', + ExternalsDescription.EXTERNALS: EMPTY_STR, + ExternalsDescription.REPO: rdatat}, + 'simp_branch' : {ExternalsDescription.REQUIRED: True, + ExternalsDescription.PATH: 'simp_branch', + ExternalsDescription.EXTERNALS: EMPTY_STR, + ExternalsDescription.REPO: rdatab}, + 'simp_opt': {ExternalsDescription.REQUIRED: False, + ExternalsDescription.PATH: 'simp_opt', + ExternalsDescription.EXTERNALS: EMPTY_STR, + ExternalsDescription.REPO: rdatat}, + 'mixed_req': {ExternalsDescription.REQUIRED: True, + ExternalsDescription.PATH: 'mixed_req', + ExternalsDescription.EXTERNALS: 'sub-ext.cfg', + ExternalsDescription.REPO: rdatam}} + + return desc + def test_cfg_v1_ok(self): """Test that a correct cfg v1 object is created by create_externals_description @@ -379,6 +413,49 @@ def test_dict(self): ext = create_externals_description(desc, model_format='dict') self.assertIsInstance(ext, ExternalsDescriptionDict) + def test_cfg_component_dict(self): + """Verify that create_externals_description works with a dictionary + """ + # create the top level externals file + desc = self.setup_dict_config() + # Check external with all repos + external = create_externals_description(desc, model_format='dict') + self.assertIsInstance(external, ExternalsDescriptionDict) + self.assertTrue('simp_tag' in external) + self.assertTrue('simp_branch' in external) + self.assertTrue('simp_opt' in external) + self.assertTrue('mixed_req' in external) + + def test_cfg_exclude_component_dict(self): + """Verify that exclude component checkout works with a dictionary + """ + # create the top level externals file + desc = self.setup_dict_config() + # Test an excluded repo + external = create_externals_description(desc, model_format='dict', + exclude=['simp_tag', + 'simp_opt']) + self.assertIsInstance(external, ExternalsDescriptionDict) + self.assertFalse('simp_tag' in external) + self.assertTrue('simp_branch' in external) + self.assertFalse('simp_opt' in external) + self.assertTrue('mixed_req' in external) + + def test_cfg_opt_component_dict(self): + """Verify that exclude component checkout works with a dictionary + """ + # create the top level externals file + desc = self.setup_dict_config() + # Test an excluded repo + external = create_externals_description(desc, model_format='dict', + components=['simp_tag', + 'simp_opt']) + self.assertIsInstance(external, ExternalsDescriptionDict) + self.assertTrue('simp_tag' in external) + self.assertFalse('simp_branch' in external) + self.assertTrue('simp_opt' in external) + self.assertFalse('mixed_req' in external) + def test_cfg_unknown_version(self): """Test that a runtime error is raised when an unknown file version is received