diff --git a/mepo.d/cmdline/parser.py b/mepo.d/cmdline/parser.py index 695668e..dc5a5cd 100644 --- a/mepo.d/cmdline/parser.py +++ b/mepo.d/cmdline/parser.py @@ -52,6 +52,13 @@ def __init(self): nargs = '?', default = 'components.yaml', help = 'default: %(default)s') + init.add_argument( + '--style', + metavar = 'style-type', + nargs = '?', + default = 'prefix', + choices = ['naked', 'prefix','postfix'], + help = 'Style of directory file, default: %(default)s, allowed options: %(choices)s') def __clone(self): clone = self.subparsers.add_parser( @@ -80,6 +87,13 @@ def __clone(self): nargs = '?', default = None, help = 'Configuration file (ignored if init already called)') + clone.add_argument( + '--style', + metavar = 'style-type', + nargs = '?', + default = 'prefix', + choices = ['naked', 'prefix','postfix'], + help = 'Style of directory file, default: %(default)s, allowed options: %(choices)s (ignored if init already called)') def __list(self): listcomps = self.subparsers.add_parser( diff --git a/mepo.d/command/init/init.py b/mepo.d/command/init/init.py index 32b9818..64c55ab 100644 --- a/mepo.d/command/init/init.py +++ b/mepo.d/command/init/init.py @@ -1,5 +1,5 @@ from state.state import MepoState def run(args): - allcomps = MepoState.initialize(args.config) - print('Initializing mepo using {}'.format(args.config)) + allcomps = MepoState.initialize(args.config,args.style) + print(f'Initializing mepo using {args.config} with {args.style} style.') diff --git a/mepo.d/command/restore-state/restore-state.py b/mepo.d/command/restore-state/restore-state.py index 1565449..dd6a025 100644 --- a/mepo.d/command/restore-state/restore-state.py +++ b/mepo.d/command/restore-state/restore-state.py @@ -1,6 +1,7 @@ import sys import time import multiprocessing as mp +import atexit from state.state import MepoState from repository.git import GitRepository @@ -11,6 +12,7 @@ def run(args): print('Checking status...'); sys.stdout.flush() allcomps = MepoState.read_state() pool = mp.Pool() + atexit.register(pool.close) result = pool.map(check_component_status, allcomps) restore_state(allcomps, result) diff --git a/mepo.d/command/status/status.py b/mepo.d/command/status/status.py index 4c2823e..fbbc5eb 100644 --- a/mepo.d/command/status/status.py +++ b/mepo.d/command/status/status.py @@ -1,6 +1,7 @@ import sys import time import multiprocessing as mp +import atexit from state.state import MepoState from repository.git import GitRepository @@ -11,6 +12,7 @@ def run(args): print('Checking status...'); sys.stdout.flush() allcomps = MepoState.read_state() pool = mp.Pool() + atexit.register(pool.close) result = pool.map(check_component_status, allcomps) print_status(allcomps, result) diff --git a/mepo.d/state/component.py b/mepo.d/state/component.py index c895fa8..7fe0667 100644 --- a/mepo.d/state/component.py +++ b/mepo.d/state/component.py @@ -5,6 +5,9 @@ from utilities import shellcmd from urllib.parse import urlparse +# This will be used to store the "final nodes" from each subrepo +original_final_node_list = [] + class MepoComponent(object): __slots__ = ['name', 'local', 'remote', 'version', 'develop', 'sparse', 'recurse_submodules', 'fixture'] @@ -83,7 +86,7 @@ def __validate_component(self, comp_name, comp_details): Component {comp_name} has {git_tag_intersection} and only one of {types_of_git_tags} are allowed.'''))) - def to_component(self, comp_name, comp_details): + def to_component(self, comp_name, comp_details, comp_style): self.name = comp_name self.fixture = comp_details.get('fixture', False) if self.fixture: @@ -96,7 +99,37 @@ def to_component(self, comp_name, comp_details): self.remote = "../"+last_url_node else: self.__validate_component(comp_name, comp_details) - self.local = comp_details['local'] + #print(f"original self.local: {comp_details['local']}") + + # Assume the flag for repostories is commercial-at + repo_flag = '@' + + # To make it easier to loop over the local path, split into a list + local_list = splitall(comp_details['local']) + + # The last node of the path is what we will decorate + last_node = local_list[-1] + + # Add that final node to a list + original_final_node_list.append(last_node) + + # Now we need to decorate all the final nodes since we can have + # nested repos with mepo + for item in original_final_node_list: + try: + # Find the index of every "final node" in a local path + # for nesting + index = local_list.index(item) + + # Decorate all final nodes + local_list[index] = decorate_node(item, repo_flag, comp_style) + except ValueError: + pass + + # Now pull the list of nodes back into a path + self.local = os.path.join(*local_list) + #print(f'final self.local: {self.local}') + self.remote = comp_details['remote'] self.develop = comp_details.get('develop', None) # develop is optional self.sparse = comp_details.get('sparse', None) # sparse is optional @@ -135,3 +168,29 @@ def get_current_remote_url(): cmd = 'git remote get-url origin' output = shellcmd.run(cmd.split(), output=True).strip() return output + +def decorate_node(item, flag, style): + item = item.replace(flag,'') + if style == 'naked': + output = item + elif style == 'prefix': + output = flag + item + elif style == 'postfix': + output = item + flag + return output + +# From https://learning.oreilly.com/library/view/python-cookbook/0596001673/ch04s16.html +def splitall(path): + allparts = [] + while 1: + parts = os.path.split(path) + if parts[0] == path: # sentinel for absolute paths + allparts.insert(0, parts[0]) + break + elif parts[1] == path: # sentinel for relative paths + allparts.insert(0, parts[1]) + break + else: + path = parts[0] + allparts.insert(0, parts[1]) + return allparts diff --git a/mepo.d/state/state.py b/mepo.d/state/state.py index 7dc13b6..fda97ab 100644 --- a/mepo.d/state/state.py +++ b/mepo.d/state/state.py @@ -53,7 +53,7 @@ def exists(cls): return False @classmethod - def initialize(cls, project_config_file): + def initialize(cls, project_config_file, directory_style): if cls.exists(): raise StateAlreadyInitializedError('mepo state already exists') input_components = ConfigFile(project_config_file).read_file() @@ -67,7 +67,7 @@ def initialize(cls, project_config_file): if num_fixture > 1: raise Exception("Only one fixture allowed") - complist.append(MepoComponent().to_component(name, comp)) + complist.append(MepoComponent().to_component(name, comp, directory_style)) cls.write_state(complist) @classmethod diff --git a/mepo.d/utest/input/components.yaml b/mepo.d/utest/input/components.yaml index 36cb7db..a56822e 100644 --- a/mepo.d/utest/input/components.yaml +++ b/mepo.d/utest/input/components.yaml @@ -1,48 +1,52 @@ +GEOSfvdycore: + fixture: true + develop: main + env: local: ./@env remote: ../ESMA_env.git - tag: v2.1.3 - develop: master + tag: v3.2.0 + develop: main cmake: local: ./@cmake remote: ../ESMA_cmake.git - tag: v3.0.2 + tag: v3.3.9 develop: develop ecbuild: local: ./@cmake/@ecbuild remote: ../ecbuild.git - tag: geos/v1.0.1 + tag: geos/v1.0.6 GMAO_Shared: local: ./src/Shared/@GMAO_Shared remote: ../GMAO_Shared.git - tag: v1.1.3 + tag: v1.3.10 sparse: ./config/GMAO_Shared.sparse - develop: master + develop: main MAPL: local: ./src/Shared/@MAPL remote: ../MAPL.git - tag: v2.1.3 + tag: v2.6.4 develop: develop FMS: local: ./src/Shared/@FMS remote: ../FMS.git - tag: geos/2019.01.01 - develop: geos/master + tag: geos/2019.01.02+noaff.6 + develop: geos/release/2019.01 FVdycoreCubed_GridComp: local: ./src/Components/@FVdycoreCubed_GridComp remote: ../FVdycoreCubed_GridComp.git - tag: v1.1.0 + tag: v1.2.12 develop: develop fvdycore: local: ./src/Components/@FVdycoreCubed_GridComp/@fvdycore remote: ../GFDL_atmos_cubed_sphere.git - tag: geos/v1.1.0 + tag: geos/v1.1.6 develop: geos/develop diff --git a/mepo.d/utest/output/compare_no_change.txt b/mepo.d/utest/output/compare_no_change.txt new file mode 100644 index 0000000..05fb2c9 --- /dev/null +++ b/mepo.d/utest/output/compare_no_change.txt @@ -0,0 +1,11 @@ +Repo | Original | Current +---------------------- | -------------------------------- | ------- +GEOSfvdycore | (b) main | (b) main +env | (t) v3.2.0 (DH) | (t) v3.2.0 (DH) +cmake | (t) v3.3.9 (DH) | (t) v3.3.9 (DH) +ecbuild | (t) geos/v1.0.6 (DH) | (t) geos/v1.0.6 (DH) +GMAO_Shared | (t) v1.3.10 (DH) | (t) v1.3.10 (DH) +MAPL | (t) v2.6.4 (DH) | (t) v2.6.4 (DH) +FMS | (t) geos/2019.01.02+noaff.6 (DH) | (t) geos/2019.01.02+noaff.6 (DH) +FVdycoreCubed_GridComp | (t) v1.2.12 (DH) | (t) v1.2.12 (DH) +fvdycore | (t) geos/v1.1.6 (DH) | (t) geos/v1.1.6 (DH) diff --git a/mepo.d/utest/output/list_output.txt b/mepo.d/utest/output/list_output.txt index 354ccee..608fc12 100644 --- a/mepo.d/utest/output/list_output.txt +++ b/mepo.d/utest/output/list_output.txt @@ -1 +1 @@ -env cmake ecbuild GMAO_Shared MAPL FMS FVdycoreCubed_GridComp fvdycore +GEOSfvdycore env cmake ecbuild GMAO_Shared MAPL FMS FVdycoreCubed_GridComp fvdycore diff --git a/mepo.d/utest/output/status_output.txt b/mepo.d/utest/output/status_output.txt index 2588e74..c2d7ea5 100644 --- a/mepo.d/utest/output/status_output.txt +++ b/mepo.d/utest/output/status_output.txt @@ -1,9 +1,10 @@ Checking status... -env | (t) v2.1.3 (DH) -cmake | (t) v3.0.2 (DH) -ecbuild | (t) geos/v1.0.1 (DH) -GMAO_Shared | (t) v1.1.3 (DH) -MAPL | (t) v2.1.3 (DH) -FMS | (t) geos/2019.01.01 (DH) -FVdycoreCubed_GridComp | (t) v1.1.0 (DH) -fvdycore | (t) geos/v1.1.0 (DH) +GEOSfvdycore | (b) main +env | (t) v3.2.0 (DH) +cmake | (t) v3.3.9 (DH) +ecbuild | (t) geos/v1.0.6 (DH) +GMAO_Shared | (t) v1.3.10 (DH) +MAPL | (t) v2.6.4 (DH) +FMS | (t) geos/2019.01.02+noaff.6 (DH) +FVdycoreCubed_GridComp | (t) v1.2.12 (DH) +fvdycore | (t) geos/v1.1.6 (DH) diff --git a/mepo.d/utest/test_mepo_commands.py b/mepo.d/utest/test_mepo_commands.py index e0f3182..38cedea 100644 --- a/mepo.d/utest/test_mepo_commands.py +++ b/mepo.d/utest/test_mepo_commands.py @@ -9,9 +9,9 @@ from input import args -from command.init import init as mepo_init -from command.clone import clone as mepo_clone -from command.list import list as mepo_list +from command.init import init as mepo_init +from command.clone import clone as mepo_clone +from command.list import list as mepo_list from command.status import status as mepo_status class TestMepoCommands(unittest.TestCase): @@ -40,6 +40,7 @@ def setUpClass(cls): cls.__checkout_fixture() cls.__copy_config_file() args.config = 'components.yaml' + args.style = 'prefix' os.chdir(cls.fixture_dir) mepo_init.run(args) args.config = None