Merge pull request #160 from GEOS-ESM/feature/mathomp4/flexible-comme…

…rcial-at

Add support for a style option for repo indicator

mepo.d/cmdline/parser.py

@@ -52,6 +52,13 @@ def __init(self):
             nargs = '?',
             default = 'components.yaml',
             help = 'default: %(default)s')
+        init.add_argument(
+            '--style',
+            metavar = 'style-type',
+            nargs = '?',
+            default = 'prefix',
+            choices = ['naked', 'prefix','postfix'],
+            help = 'Style of directory file, default: %(default)s, allowed options: %(choices)s')
 
     def __clone(self):
         clone = self.subparsers.add_parser(
@@ -80,6 +87,13 @@ def __clone(self):
             nargs = '?',
             default = None,
             help = 'Configuration file (ignored if init already called)')
+        clone.add_argument(
+            '--style',
+            metavar = 'style-type',
+            nargs = '?',
+            default = 'prefix',
+            choices = ['naked', 'prefix','postfix'],
+            help = 'Style of directory file, default: %(default)s, allowed options: %(choices)s (ignored if init already called)')
 
     def __list(self):
         listcomps = self.subparsers.add_parser(

mepo.d/command/init/init.py

@@ -1,5 +1,5 @@
 from state.state import MepoState
 
 def run(args):
-    allcomps = MepoState.initialize(args.config)
-    print('Initializing mepo using {}'.format(args.config))
+    allcomps = MepoState.initialize(args.config,args.style)
+    print(f'Initializing mepo using {args.config} with {args.style} style.')

mepo.d/command/restore-state/restore-state.py

@@ -1,6 +1,7 @@
 import sys
 import time
 import multiprocessing as mp
+import atexit
 
 from state.state import MepoState
 from repository.git import GitRepository
@@ -11,6 +12,7 @@ def run(args):
     print('Checking status...'); sys.stdout.flush()
     allcomps = MepoState.read_state()
     pool = mp.Pool()
+    atexit.register(pool.close)
     result = pool.map(check_component_status, allcomps)
     restore_state(allcomps, result)
 

mepo.d/command/status/status.py

@@ -1,6 +1,7 @@
 import sys
 import time
 import multiprocessing as mp
+import atexit
 
 from state.state import MepoState
 from repository.git import GitRepository
@@ -11,6 +12,7 @@ def run(args):
     print('Checking status...'); sys.stdout.flush()
     allcomps = MepoState.read_state()
     pool = mp.Pool()
+    atexit.register(pool.close)
     result = pool.map(check_component_status, allcomps)
     print_status(allcomps, result)
 

mepo.d/state/component.py

@@ -5,6 +5,9 @@
 from utilities import shellcmd
 from urllib.parse import urlparse
 
+# This will be used to store the "final nodes" from each subrepo
+original_final_node_list = []
+
 class MepoComponent(object):
 
     __slots__ = ['name', 'local', 'remote', 'version', 'develop', 'sparse', 'recurse_submodules', 'fixture']
@@ -83,7 +86,7 @@ def __validate_component(self, comp_name, comp_details):
                 Component {comp_name} has {git_tag_intersection} and only one of
                 {types_of_git_tags} are allowed.''')))
 
-    def to_component(self, comp_name, comp_details):
+    def to_component(self, comp_name, comp_details, comp_style):
         self.name = comp_name
         self.fixture = comp_details.get('fixture', False)
         if self.fixture:
@@ -96,7 +99,37 @@ def to_component(self, comp_name, comp_details):
             self.remote = "../"+last_url_node
         else:
             self.__validate_component(comp_name, comp_details)
-            self.local = comp_details['local']
+            #print(f"original self.local: {comp_details['local']}")
+
+            # Assume the flag for repostories is commercial-at
+            repo_flag = '@'
+
+            # To make it easier to loop over the local path, split into a list
+            local_list = splitall(comp_details['local'])
+
+            # The last node of the path is what we will decorate
+            last_node = local_list[-1]
+
+            # Add that final node to a list
+            original_final_node_list.append(last_node)
+
+            # Now we need to decorate all the final nodes since we can have
+            # nested repos with mepo
+            for item in original_final_node_list:
+                try:
+                    # Find the index of every "final node" in a local path
+                    # for nesting
+                    index = local_list.index(item)
+
+                    # Decorate all final nodes
+                    local_list[index] = decorate_node(item, repo_flag, comp_style)
+                except ValueError:
+                    pass
+
+            # Now pull the list of nodes back into a path
+            self.local = os.path.join(*local_list)
+            #print(f'final self.local: {self.local}')
+
             self.remote = comp_details['remote']
         self.develop = comp_details.get('develop', None) # develop is optional
         self.sparse = comp_details.get('sparse', None) # sparse is optional
@@ -135,3 +168,29 @@ def get_current_remote_url():
     cmd = 'git remote get-url origin'
     output = shellcmd.run(cmd.split(), output=True).strip()
     return output
+
+def decorate_node(item, flag, style):
+    item = item.replace(flag,'')
+    if style == 'naked':
+        output = item
+    elif style == 'prefix':
+        output = flag + item
+    elif style == 'postfix':
+        output = item + flag
+    return output
+
+# From https://learning.oreilly.com/library/view/python-cookbook/0596001673/ch04s16.html
+def splitall(path):
+    allparts = []
+    while 1:
+        parts = os.path.split(path)
+        if parts[0] == path:  # sentinel for absolute paths
+            allparts.insert(0, parts[0])
+            break
+        elif parts[1] == path: # sentinel for relative paths
+            allparts.insert(0, parts[1])
+            break
+        else:
+            path = parts[0]
+            allparts.insert(0, parts[1])
+    return allparts

mepo.d/state/state.py

@@ -53,7 +53,7 @@ def exists(cls):
             return False
 
     @classmethod
-    def initialize(cls, project_config_file):
+    def initialize(cls, project_config_file, directory_style):
         if cls.exists():
             raise StateAlreadyInitializedError('mepo state already exists')
         input_components = ConfigFile(project_config_file).read_file()
@@ -67,7 +67,7 @@ def initialize(cls, project_config_file):
             if num_fixture > 1:
                 raise Exception("Only one fixture allowed")
 
-            complist.append(MepoComponent().to_component(name, comp))
+            complist.append(MepoComponent().to_component(name, comp, directory_style))
         cls.write_state(complist)
 
     @classmethod

mepo.d/utest/input/components.yaml

@@ -1,48 +1,52 @@
+GEOSfvdycore:
+  fixture: true
+  develop: main
+
 env:
   local: ./@env
   remote: ../ESMA_env.git
-  tag: v2.1.3
-  develop: master
+  tag: v3.2.0
+  develop: main
 
 cmake:
   local: ./@cmake
   remote: ../ESMA_cmake.git
-  tag: v3.0.2
+  tag: v3.3.9
   develop: develop
 
 ecbuild:
   local: ./@cmake/@ecbuild
   remote: ../ecbuild.git
-  tag: geos/v1.0.1
+  tag: geos/v1.0.6
 
 GMAO_Shared:
   local: ./src/Shared/@GMAO_Shared
   remote: ../GMAO_Shared.git
-  tag: v1.1.3
+  tag: v1.3.10
   sparse: ./config/GMAO_Shared.sparse
-  develop: master
+  develop: main
 
 MAPL:
   local: ./src/Shared/@MAPL
   remote: ../MAPL.git
-  tag: v2.1.3
+  tag: v2.6.4
   develop: develop
 
 FMS:
   local: ./src/Shared/@FMS
   remote: ../FMS.git
-  tag: geos/2019.01.01
-  develop: geos/master
+  tag: geos/2019.01.02+noaff.6
+  develop: geos/release/2019.01
 
 FVdycoreCubed_GridComp:
   local: ./src/Components/@FVdycoreCubed_GridComp
   remote: ../FVdycoreCubed_GridComp.git
-  tag: v1.1.0
+  tag: v1.2.12
   develop: develop
 
 fvdycore:
   local: ./src/Components/@FVdycoreCubed_GridComp/@fvdycore
   remote: ../GFDL_atmos_cubed_sphere.git
-  tag: geos/v1.1.0
+  tag: geos/v1.1.6
   develop: geos/develop
 

mepo.d/utest/output/compare_no_change.txt

@@ -0,0 +1,11 @@
+Repo                   | Original                         | Current
+---------------------- | -------------------------------- | -------
+GEOSfvdycore           | (b) main                         | (b) main
+env                    | (t) v3.2.0 (DH)                  | (t) v3.2.0 (DH)
+cmake                  | (t) v3.3.9 (DH)                  | (t) v3.3.9 (DH)
+ecbuild                | (t) geos/v1.0.6 (DH)             | (t) geos/v1.0.6 (DH)
+GMAO_Shared            | (t) v1.3.10 (DH)                 | (t) v1.3.10 (DH)
+MAPL                   | (t) v2.6.4 (DH)                  | (t) v2.6.4 (DH)
+FMS                    | (t) geos/2019.01.02+noaff.6 (DH) | (t) geos/2019.01.02+noaff.6 (DH)
+FVdycoreCubed_GridComp | (t) v1.2.12 (DH)                 | (t) v1.2.12 (DH)
+fvdycore               | (t) geos/v1.1.6 (DH)             | (t) geos/v1.1.6 (DH)

mepo.d/utest/output/list_output.txt

@@ -1 +1 @@
-env cmake ecbuild GMAO_Shared MAPL FMS FVdycoreCubed_GridComp fvdycore 
+GEOSfvdycore env cmake ecbuild GMAO_Shared MAPL FMS FVdycoreCubed_GridComp fvdycore 

mepo.d/utest/output/status_output.txt

@@ -1,9 +1,10 @@
 Checking status...
-env                    | (t) v2.1.3 (DH)
-cmake                  | (t) v3.0.2 (DH)
-ecbuild                | (t) geos/v1.0.1 (DH)
-GMAO_Shared            | (t) v1.1.3 (DH)
-MAPL                   | (t) v2.1.3 (DH)
-FMS                    | (t) geos/2019.01.01 (DH)
-FVdycoreCubed_GridComp | (t) v1.1.0 (DH)
-fvdycore               | (t) geos/v1.1.0 (DH)
+GEOSfvdycore           | (b) main
+env                    | (t) v3.2.0 (DH)
+cmake                  | (t) v3.3.9 (DH)
+ecbuild                | (t) geos/v1.0.6 (DH)
+GMAO_Shared            | (t) v1.3.10 (DH)
+MAPL                   | (t) v2.6.4 (DH)
+FMS                    | (t) geos/2019.01.02+noaff.6 (DH)
+FVdycoreCubed_GridComp | (t) v1.2.12 (DH)
+fvdycore               | (t) geos/v1.1.6 (DH)

mepo.d/utest/test_mepo_commands.py

@@ -9,9 +9,9 @@
 
 from input import args
 
-from command.init import init as mepo_init
-from command.clone import clone as mepo_clone
-from command.list import list as mepo_list
+from command.init   import init   as mepo_init
+from command.clone  import clone  as mepo_clone
+from command.list   import list   as mepo_list
 from command.status import status as mepo_status
 
 class TestMepoCommands(unittest.TestCase):
@@ -40,6 +40,7 @@ def setUpClass(cls):
         cls.__checkout_fixture()
         cls.__copy_config_file()
         args.config = 'components.yaml'
+        args.style = 'prefix'
         os.chdir(cls.fixture_dir)
         mepo_init.run(args)
         args.config = None