Merge pull request ESCOMP#152 from jedwards4b/sort_by_local_path

Sort by local path

.travis.yml

@@ -1,7 +1,6 @@
 language: python
 os: linux
-python: 
-  - "2.7"
+python:
   - "3.4"
   - "3.5"
   - "3.6"

manic/checkout.py

@@ -380,14 +380,14 @@ def main(args):
 
     if args.status:
         # user requested status-only
-        for comp in sorted(tree_status.keys()):
+        for comp in sorted(tree_status):
             tree_status[comp].log_status_message(args.verbose)
     else:
         # checkout / update the external repositories.
         safe_to_update = check_safe_to_update_repos(tree_status)
         if not safe_to_update:
             # print status
-            for comp in sorted(tree_status.keys()):
+            for comp in sorted(tree_status):
                 tree_status[comp].log_status_message(args.verbose)
             # exit gracefully
             msg = """The external repositories labeled with 'M' above are not in a clean state.

manic/sourcetree.py

@@ -331,12 +331,14 @@ def checkout(self, verbosity, load_all, load_comp=None):
             printlog('Checking out externals: ', end='')
 
         if load_all:
-            load_comps = self._all_components.keys()
+            tmp_comps = self._all_components.keys()
         elif load_comp is not None:
-            load_comps = [load_comp]
+            tmp_comps = [load_comp]
         else:
-            load_comps = self._required_compnames
-
+            tmp_comps = self._required_compnames
+        # Sort by path so that if paths are nested the
+        # parent repo is checked out first.
+        load_comps = sorted(tmp_comps, key=lambda comp: self._all_components[comp].get_local_path())
         # checkout the primary externals
         for comp in load_comps:
             if verbosity < VERBOSITY_VERBOSE:
@@ -346,8 +348,6 @@ def checkout(self, verbosity, load_all, load_comp=None):
                 # output a newline
                 printlog(EMPTY_STR)
             self._all_components[comp].checkout(verbosity, load_all)
-        printlog('')
-
-        # now give each external an opportunitity to checkout it's externals.
-        for comp in load_comps:
+            # now give each external an opportunitity to checkout it's externals.
             self._all_components[comp].checkout_externals(verbosity, load_all)
+        printlog('')

test/test_sys_checkout.py

@@ -85,6 +85,8 @@
 CFG_SUB_NAME = 'sub-externals.cfg'
 README_NAME = 'readme.txt'
 REMOTE_BRANCH_FEATURE2 = 'feature2'
+NESTED_NAME = ['./fred', './fred/wilma', './fred/wilma/barney']
+
 
 SVN_TEST_REPO = 'https://github.com/escomp/cesm'
 
@@ -160,6 +162,23 @@ def container_simple_required(self, dest_dir):
 
         self.write_config(dest_dir)
 
+    def container_nested_required(self, dest_dir, order):
+        """Create a container externals file with only simple externals.
+
+        """
+        self.create_config()
+        self.create_section(SIMPLE_REPO_NAME, 'simp_tag', nested=True,
+                            tag='tag1', path=NESTED_NAME[order[0]])
+
+        self.create_section(SIMPLE_REPO_NAME, 'simp_branch', nested=True,
+                            branch=REMOTE_BRANCH_FEATURE2, path=NESTED_NAME[order[1]])
+
+        self.create_section(SIMPLE_REPO_NAME, 'simp_hash', nested=True,
+                            ref_hash='60b1cc1a38d63', path=NESTED_NAME[order[2]])
+
+        self.write_config(dest_dir)
+
+
     def container_simple_optional(self, dest_dir):
         """Create a container externals file with optional simple externals
 
@@ -261,7 +280,7 @@ def create_metadata(self):
     def create_section(self, repo_type, name, tag='', branch='',
                        ref_hash='', required=True, path=EXTERNALS_NAME,
                        externals='', repo_path=None, from_submodule=False,
-                       sparse=''):
+                       sparse='', nested=False):
         # pylint: disable=too-many-branches
         """Create a config section with autofilling some items and handling
         optional items.
@@ -270,8 +289,11 @@ def create_section(self, repo_type, name, tag='', branch='',
         # pylint: disable=R0913
         self._config.add_section(name)
         if not from_submodule:
-            self._config.set(name, ExternalsDescription.PATH,
-                             os.path.join(path, name))
+            if nested:
+                self._config.set(name, ExternalsDescription.PATH, path)
+            else:
+                self._config.set(name, ExternalsDescription.PATH,
+                                 os.path.join(path, name))
 
         self._config.set(name, ExternalsDescription.PROTOCOL,
                          ExternalsDescription.PROTOCOL_GIT)
@@ -671,10 +693,16 @@ def _check_simple_tag_empty(self, tree, directory=EXTERNALS_NAME):
         name = './{0}/simp_tag'.format(directory)
         self._check_generic_empty_default_required(tree, name)
 
+    def _check_nested_tag_empty(self, tree, name=EXTERNALS_NAME):
+        self._check_generic_empty_default_required(tree, name)
+
     def _check_simple_tag_ok(self, tree, directory=EXTERNALS_NAME):
         name = './{0}/simp_tag'.format(directory)
         self._check_generic_ok_clean_required(tree, name)
 
+    def _check_nested_tag_ok(self, tree, name=EXTERNALS_NAME):
+        self._check_generic_ok_clean_required(tree, name)
+
     def _check_simple_tag_dirty(self, tree, directory=EXTERNALS_NAME):
         name = './{0}/simp_tag'.format(directory)
         self._check_generic_ok_dirty_required(tree, name)
@@ -687,10 +715,16 @@ def _check_simple_branch_empty(self, tree, directory=EXTERNALS_NAME):
         name = './{0}/simp_branch'.format(directory)
         self._check_generic_empty_default_required(tree, name)
 
+    def _check_nested_branch_empty(self, tree, name=EXTERNALS_NAME):
+        self._check_generic_empty_default_required(tree, name)
+
     def _check_simple_branch_ok(self, tree, directory=EXTERNALS_NAME):
         name = './{0}/simp_branch'.format(directory)
         self._check_generic_ok_clean_required(tree, name)
 
+    def _check_nested_branch_ok(self, tree, name=EXTERNALS_NAME):
+        self._check_generic_ok_clean_required(tree, name)
+
     def _check_simple_branch_modified(self, tree, directory=EXTERNALS_NAME):
         name = './{0}/simp_branch'.format(directory)
         self._check_generic_modified_ok_required(tree, name)
@@ -699,10 +733,16 @@ def _check_simple_hash_empty(self, tree, directory=EXTERNALS_NAME):
         name = './{0}/simp_hash'.format(directory)
         self._check_generic_empty_default_required(tree, name)
 
+    def _check_nested_hash_empty(self, tree, name=EXTERNALS_NAME):
+        self._check_generic_empty_default_required(tree, name)
+
     def _check_simple_hash_ok(self, tree, directory=EXTERNALS_NAME):
         name = './{0}/simp_hash'.format(directory)
         self._check_generic_ok_clean_required(tree, name)
 
+    def _check_nested_hash_ok(self, tree, name=EXTERNALS_NAME):
+        self._check_generic_ok_clean_required(tree, name)
+
     def _check_simple_hash_modified(self, tree, directory=EXTERNALS_NAME):
         name = './{0}/simp_hash'.format(directory)
         self._check_generic_modified_ok_required(tree, name)
@@ -754,19 +794,38 @@ def _check_container_simple_required_pre_checkout(self, overall, tree):
         self._check_simple_branch_empty(tree)
         self._check_simple_hash_empty(tree)
 
+    def _check_container_nested_required_pre_checkout(self, overall, tree, order):
+        self.assertEqual(overall, 0)
+        self._check_nested_tag_empty(tree, name=NESTED_NAME[order[0]])
+        self._check_nested_branch_empty(tree, name=NESTED_NAME[order[1]])
+        self._check_nested_hash_empty(tree, name=NESTED_NAME[order[2]])
+
     def _check_container_simple_required_checkout(self, overall, tree):
         # Note, this is the internal tree status just before checkout
         self.assertEqual(overall, 0)
         self._check_simple_tag_empty(tree)
         self._check_simple_branch_empty(tree)
         self._check_simple_hash_empty(tree)
 
+    def _check_container_nested_required_checkout(self, overall, tree, order):
+        # Note, this is the internal tree status just before checkout
+        self.assertEqual(overall, 0)
+        self._check_nested_tag_empty(tree, name=NESTED_NAME[order[0]])
+        self._check_nested_branch_empty(tree, name=NESTED_NAME[order[1]])
+        self._check_nested_hash_empty(tree, name=NESTED_NAME[order[2]])
+
     def _check_container_simple_required_post_checkout(self, overall, tree):
         self.assertEqual(overall, 0)
         self._check_simple_tag_ok(tree)
         self._check_simple_branch_ok(tree)
         self._check_simple_hash_ok(tree)
 
+    def _check_container_nested_required_post_checkout(self, overall, tree, order):
+        self.assertEqual(overall, 0)
+        self._check_nested_tag_ok(tree, name=NESTED_NAME[order[0]])
+        self._check_nested_branch_ok(tree, name=NESTED_NAME[order[1]])
+        self._check_nested_hash_ok(tree, name=NESTED_NAME[order[2]])
+
     def _check_container_simple_required_out_of_sync(self, overall, tree):
         self.assertEqual(overall, 0)
         self._check_simple_tag_modified(tree)
@@ -964,6 +1023,37 @@ def test_container_simple_required(self):
                                                 self.status_args)
         self._check_container_simple_required_post_checkout(overall, tree)
 
+    def test_container_nested_required(self):
+        """Verify that a container with nested subrepos
+        generates the correct initial status.
+        Tests over all possible permutations
+        """
+
+        orders = [[0, 1, 2], [1, 2, 0], [2, 0, 1],
+                  [0, 2, 1], [2, 1, 0], [1, 0, 2]]
+        for n, order in enumerate(orders):
+            # create repo
+            dest_dir = os.path.join(os.environ[MANIC_TEST_TMP_REPO_ROOT],
+                                  self._test_id, "test"+str(n))
+            under_test_dir = self.setup_test_repo(CONTAINER_REPO_NAME,
+                                                  dest_dir_in=dest_dir)
+            self._generator.container_nested_required(under_test_dir, order)
+
+            # status of empty repo
+            overall, tree = self.execute_cmd_in_dir(under_test_dir,
+                                                    self.status_args)
+            self._check_container_nested_required_pre_checkout(overall, tree, order)
+
+            # checkout
+            overall, tree = self.execute_cmd_in_dir(under_test_dir,
+                                                    self.checkout_args)
+            self._check_container_nested_required_checkout(overall, tree, order)
+
+            # status clean checked out
+            overall, tree = self.execute_cmd_in_dir(under_test_dir,
+                                                    self.status_args)
+            self._check_container_nested_required_post_checkout(overall, tree, order)
+
     def test_container_simple_optional(self):
         """Verify that container with an optional simple subrepos
         generates the correct initial status.
@@ -1591,7 +1681,7 @@ def setUp(self):
         """
 
         # Run the basic setup
-        super(TestSubrepoCheckout, self).setUp()
+        super().setUp()
         # create test repo
         # We need to do this here (rather than have a static repo) because
         # git submodules do not allow for variables in .gitmodules files