From 98a810db57659f8bbf02af64ae2e753d5537be5c Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Thu, 29 Mar 2018 10:58:34 -0600 Subject: [PATCH 1/8] add a components arg to checkout only select components --- manic/checkout.py | 18 ++++++++++++++---- manic/externals_description.py | 21 ++++++++++++++------- manic/sourcetree.py | 2 +- 3 files changed, 29 insertions(+), 12 deletions(-) diff --git a/manic/checkout.py b/manic/checkout.py index 98bf3f3..b84e14c 100755 --- a/manic/checkout.py +++ b/manic/checkout.py @@ -20,7 +20,7 @@ from manic.externals_description import read_externals_description_file from manic.externals_status import check_safe_to_update_repos from manic.sourcetree import SourceTree -from manic.utils import printlog +from manic.utils import printlog, fatal_error from manic.global_constants import VERSION_SEPERATOR, LOG_FILE_NAME if sys.hexversion < 0x02070000: @@ -243,6 +243,10 @@ def commandline_arguments(args=None): # # user options # + parser.add_argument("components", nargs="*", + help="Specific component(s) to checkout. By default" + "all required externals are checked out.") + parser.add_argument('-e', '--externals', nargs='?', default='Externals.cfg', help='The externals description filename. ' @@ -311,12 +315,18 @@ def main(args): logging.info('Beginning of %s', program_name) load_all = False - if args.optional: + if args.optional or args.components: load_all = True root_dir = os.path.abspath(os.getcwd()) external_data = read_externals_description_file(root_dir, args.externals) - external = create_externals_description(external_data) + external = create_externals_description(external_data, components=args.components) + + if args.components: + for comp in args.components: + if comp not in external.keys(): + fatal_error("No component {} found in {}".format(comp, args.externals)) + source_tree = SourceTree(root_dir, external) printlog('Checking status of externals: ', end='') @@ -354,7 +364,7 @@ def main(args): printlog(msg) printlog('-' * 70) else: - source_tree.checkout(args.verbose, load_all) + source_tree.checkout(args.verbose, load_all, load_comp=args.components) printlog('') logging.info('%s completed without exceptions.', program_name) diff --git a/manic/externals_description.py b/manic/externals_description.py index 65569da..e370958 100644 --- a/manic/externals_description.py +++ b/manic/externals_description.py @@ -91,16 +91,16 @@ def read_externals_description_file(root_dir, file_name): return externals_description -def create_externals_description(model_data, model_format='cfg'): +def create_externals_description(model_data, model_format='cfg', components=None): """Create the a externals description object from the provided data """ externals_description = None if model_format == 'dict': - externals_description = ExternalsDescriptionDict(model_data, ) + externals_description = ExternalsDescriptionDict(model_data, components=components) elif model_format == 'cfg': major, _, _ = get_cfg_schema_version(model_data) if major == 1: - externals_description = ExternalsDescriptionConfigV1(model_data) + externals_description = ExternalsDescriptionConfigV1(model_data, components=components) else: msg = ('Externals description file has unsupported schema ' 'version "{0}".'.format(major)) @@ -419,7 +419,7 @@ class ExternalsDescriptionDict(ExternalsDescription): """ - def __init__(self, model_data): + def __init__(self, model_data, components=None): """Parse a native dictionary into a externals description. """ ExternalsDescription.__init__(self) @@ -430,6 +430,11 @@ def __init__(self, model_data): self._input_minor = 0 self._input_patch = 0 self._verify_schema_version() + if components: + for k in model_data.items(): + if k not in components: + del model_data[k] + self.update(model_data) self._check_user_input() @@ -440,7 +445,7 @@ class ExternalsDescriptionConfigV1(ExternalsDescription): """ - def __init__(self, model_data): + def __init__(self, model_data, components=None): """Convert the xml into a standardized dict that can be used to construct the source objects @@ -453,7 +458,7 @@ def __init__(self, model_data): get_cfg_schema_version(model_data) self._verify_schema_version() self._remove_metadata(model_data) - self._parse_cfg(model_data) + self._parse_cfg(model_data, components=components) self._check_user_input() @staticmethod @@ -465,7 +470,7 @@ def _remove_metadata(model_data): """ model_data.remove_section(DESCRIPTION_SECTION) - def _parse_cfg(self, cfg_data): + def _parse_cfg(self, cfg_data, components=None): """Parse a config_parser object into a externals description. """ def list_to_dict(input_list, convert_to_lower_case=True): @@ -482,6 +487,8 @@ def list_to_dict(input_list, convert_to_lower_case=True): for section in cfg_data.sections(): name = config_string_cleaner(section.lower().strip()) + if components and name not in components: + continue self[name] = {} self[name].update(list_to_dict(cfg_data.items(section))) self[name][self.REPO] = {} diff --git a/manic/sourcetree.py b/manic/sourcetree.py index a97851e..dff91dc 100644 --- a/manic/sourcetree.py +++ b/manic/sourcetree.py @@ -19,7 +19,7 @@ class _External(object): """ - _External represents an external object in side a SourceTree + _External represents an external object inside a SourceTree """ # pylint: disable=R0902 From 987df5a3aba092b818d7459d71da8e51304822cd Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Tue, 3 Apr 2018 17:38:44 -0600 Subject: [PATCH 2/8] only use components if populated --- manic/checkout.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/manic/checkout.py b/manic/checkout.py index b84e14c..fc3c68c 100755 --- a/manic/checkout.py +++ b/manic/checkout.py @@ -364,7 +364,11 @@ def main(args): printlog(msg) printlog('-' * 70) else: - source_tree.checkout(args.verbose, load_all, load_comp=args.components) + if args.components: + for comp in args.components: + source_tree.checkout(args.verbose, load_all, load_comp=args.comp) + else: + source_tree.checkout(args.verbose, load_all) printlog('') logging.info('%s completed without exceptions.', program_name) From 91d4851de0b4d1b2d37a2e8c21c4cc86fdd90d83 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Tue, 3 Apr 2018 18:08:42 -0600 Subject: [PATCH 3/8] fix pylint issue --- manic/checkout.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/manic/checkout.py b/manic/checkout.py index fc3c68c..063080a 100755 --- a/manic/checkout.py +++ b/manic/checkout.py @@ -322,10 +322,9 @@ def main(args): external_data = read_externals_description_file(root_dir, args.externals) external = create_externals_description(external_data, components=args.components) - if args.components: - for comp in args.components: - if comp not in external.keys(): - fatal_error("No component {} found in {}".format(comp, args.externals)) + for comp in args.components: + if comp not in external.keys(): + fatal_error("No component {} found in {}".format(comp, args.externals)) source_tree = SourceTree(root_dir, external) @@ -364,11 +363,10 @@ def main(args): printlog(msg) printlog('-' * 70) else: - if args.components: - for comp in args.components: - source_tree.checkout(args.verbose, load_all, load_comp=args.comp) - else: + if not args.components: source_tree.checkout(args.verbose, load_all) + for comp in args.components: + source_tree.checkout(args.verbose, load_all, load_comp=args.comp) printlog('') logging.info('%s completed without exceptions.', program_name) From bf52ac615801bc04ad85ad1ed3a0b4a9cae15dcb Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Tue, 3 Apr 2018 20:54:32 -0600 Subject: [PATCH 4/8] add a test --- manic/checkout.py | 2 +- test/test_sys_checkout.py | 26 ++++++++++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/manic/checkout.py b/manic/checkout.py index 063080a..72f0e63 100755 --- a/manic/checkout.py +++ b/manic/checkout.py @@ -366,7 +366,7 @@ def main(args): if not args.components: source_tree.checkout(args.verbose, load_all) for comp in args.components: - source_tree.checkout(args.verbose, load_all, load_comp=args.comp) + source_tree.checkout(args.verbose, load_all, load_comp=comp) printlog('') logging.info('%s completed without exceptions.', program_name) diff --git a/test/test_sys_checkout.py b/test/test_sys_checkout.py index 443dd70..efd1c53 100644 --- a/test/test_sys_checkout.py +++ b/test/test_sys_checkout.py @@ -752,6 +752,12 @@ def _check_container_full_pre_checkout(self, overall, tree): self._check_simple_opt_empty(tree) self._check_mixed_ext_branch_required_pre_checkout(overall, tree) + def _check_container_component_post_checkout(self, overall, tree): + self.assertEqual(overall, 0) + self._check_simple_opt_ok(tree) + self._check_simple_tag_empty(tree) + self._check_simple_branch_empty(tree) + def _check_container_full_post_checkout(self, overall, tree): self.assertEqual(overall, 0) self._check_simple_tag_ok(tree) @@ -1219,6 +1225,26 @@ def test_container_full(self): self.status_args) self._check_container_full_post_checkout(overall, tree) + def test_container_component(self): + """Verify that optional component checkout works + """ + # create the test repository + under_test_dir = self.setup_test_repo(CONTAINER_REPO_NAME) + + # create the top level externals file + self._generator.container_full(under_test_dir) + + # inital checkout + checkout_args = ['simp_opt'] + checkout_args.extend(self.checkout_args) + overall, tree = self.execute_cmd_in_dir(under_test_dir, + checkout_args) + + overall, tree = self.execute_cmd_in_dir(under_test_dir, + self.status_args) + self._check_container_component_post_checkout(overall, tree) + + def test_mixed_simple(self): """Verify that a mixed use repo can serve as a 'full' container, pulling in a set of externals and a seperate set of sub-externals. From f78d60f3c5e2f02ff6ccc23076a321640c07e537 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Tue, 3 Apr 2018 21:21:29 -0600 Subject: [PATCH 5/8] another test --- test/test_sys_checkout.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/test/test_sys_checkout.py b/test/test_sys_checkout.py index efd1c53..e577fb3 100644 --- a/test/test_sys_checkout.py +++ b/test/test_sys_checkout.py @@ -758,6 +758,12 @@ def _check_container_component_post_checkout(self, overall, tree): self._check_simple_tag_empty(tree) self._check_simple_branch_empty(tree) + def _check_container_component_post_checkout2(self, overall, tree): + self.assertEqual(overall, 0) + self._check_simple_opt_ok(tree) + self._check_simple_tag_empty(tree) + self._check_simple_branch_ok(tree) + def _check_container_full_post_checkout(self, overall, tree): self.assertEqual(overall, 0) self._check_simple_tag_ok(tree) @@ -1243,7 +1249,12 @@ def test_container_component(self): overall, tree = self.execute_cmd_in_dir(under_test_dir, self.status_args) self._check_container_component_post_checkout(overall, tree) - + checkout_args.append('simp_branch') + overall, tree = self.execute_cmd_in_dir(under_test_dir, + checkout_args) + overall, tree = self.execute_cmd_in_dir(under_test_dir, + self.status_args) + self._check_container_component_post_checkout2(overall, tree) def test_mixed_simple(self): """Verify that a mixed use repo can serve as a 'full' container, From 4fdf1802668dae6bb43aac32e09722a1cd9a655e Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Wed, 4 Apr 2018 06:13:39 -0600 Subject: [PATCH 6/8] one more test --- test/test_sys_checkout.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/test/test_sys_checkout.py b/test/test_sys_checkout.py index e577fb3..9ebfb0a 100644 --- a/test/test_sys_checkout.py +++ b/test/test_sys_checkout.py @@ -1240,9 +1240,16 @@ def test_container_component(self): # create the top level externals file self._generator.container_full(under_test_dir) - # inital checkout + # inital checkout, first try a nonexistant component argument noref + checkout_args = ['simp_opt', 'noref'] + checkout_args.extend(self.checkout_args) + + with self.assertRaises(RuntimeError): + self.execute_cmd_in_dir(under_test_dir, checkout_args) + checkout_args = ['simp_opt'] checkout_args.extend(self.checkout_args) + overall, tree = self.execute_cmd_in_dir(under_test_dir, checkout_args) From c1b5b09d9d224fd63a26fe6af85a18e37212fd7b Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Tue, 10 Apr 2018 14:16:35 -0600 Subject: [PATCH 7/8] remove unneeded logic --- manic/checkout.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manic/checkout.py b/manic/checkout.py index 72f0e63..007aa6e 100755 --- a/manic/checkout.py +++ b/manic/checkout.py @@ -315,7 +315,7 @@ def main(args): logging.info('Beginning of %s', program_name) load_all = False - if args.optional or args.components: + if args.optional: load_all = True root_dir = os.path.abspath(os.getcwd()) From 3f4c88fe2d8c3868fc4051a71f371f1c041ee41a Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Tue, 10 Apr 2018 14:20:31 -0600 Subject: [PATCH 8/8] fix comment --- manic/externals_description.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manic/externals_description.py b/manic/externals_description.py index e370958..39a64d1 100644 --- a/manic/externals_description.py +++ b/manic/externals_description.py @@ -446,7 +446,7 @@ class ExternalsDescriptionConfigV1(ExternalsDescription): """ def __init__(self, model_data, components=None): - """Convert the xml into a standardized dict that can be used to + """Convert the config data into a standardized dict that can be used to construct the source objects """