From 60484551262ec6255f6d68d8189ec382c1987574 Mon Sep 17 00:00:00 2001 From: James Edwards Date: Thu, 18 Jan 2024 09:38:09 -0700 Subject: [PATCH] add fleximod test workflow --- .github/workflows/fleximod-test.yml | 16 ++++++ .gitmodules | 54 ++++++++++++++++++- bin/git-fleximod | 55 ++++++++++++-------- lib/python/site-packages/fleximod/version.py | 2 +- 4 files changed, 103 insertions(+), 24 deletions(-) create mode 100644 .github/workflows/fleximod-test.yml diff --git a/.github/workflows/fleximod-test.yml b/.github/workflows/fleximod-test.yml new file mode 100644 index 000000000..a324391b1 --- /dev/null +++ b/.github/workflows/fleximod-test.yml @@ -0,0 +1,16 @@ +on: + push: + branches: + - master + - remove_manage_externals + pull_request: + branches: + - master + - remove_manage_externals +jobs: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: git-fleximod test + run: | + $GITHUB_WORKSPACE/bin/git-fleximod test diff --git a/.gitmodules b/.gitmodules index 12f2e8d2f..7c34eccfa 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,137 +1,189 @@ +# This is a git submodule file with additional support for +# git-fleximod (https://github.com/jedwards4b/git-fleximod) +# +# The additional flags supported by git-fleximod are +# fxtag - the tag associated with the submodule, this tag can be tested for +# consistancy with the submodule hash using git-fleximod status +# the hash can be updated to the tag using git-fleximod update +# +# fxrequired - indicates if a given submodule should be checked out on install +# submoudules can be toplevel or internal and required or optional +# toplevel means that the submodule should only be checked out if the +# module is the toplevel of the git repo (is not a submodule itself) +# internal means that the submodule is needed by the component whether +# the component is toplevel or the submodule of another repo +# required means that the submodule should always be checked out +# optional means that the submodule should only be checked out if the +# optional flag is provided to git-fleximod or the submodule name is +# explicitly listed on the git-fleximod command line. +# +# fxsparse - this is a path to a git sparse checkout file indicating that the +# submodule should be checked out in sparse mode +# +# fxurl - this field is used by git-fleximod test to insure that the url is pointing +# to the official url of the repo and not to an unofficial fork. +# It is intended for use of github workflows to test commits to protected +# repository branches. +# + + [submodule "ccs_config"] path = ccs_config url = https://github.com/ESMCI/ccs_config_cesm.git + fxurl = https://github.com/ESMCI/ccs_config_cesm.git fxtag = ccs_config_cesm0.0.88 fxrequired = T:T [submodule "share"] path = share url = https://github.com/ESCOMP/CESM_share + fxurl = https://github.com/ESCOMP/CESM_share fxtag = share1.0.18 fxrequired = T:T [submodule "cime"] path = cime url = https://github.com/jedwards4b/cime + fxurl = https://github.com/ESMCI/cime fxtag = cime6.0.198_rme04 fxrequired = T:T [submodule "mct"] path = libraries/mct url = https://github.com/MCSclimate/MCT + fxurl = https://github.com/MCSclimate/MCT fxrequired = T:T fxtag = MCT_2.11.0 [submodule "mpi-serial"] path = libraries/mpi-serial url = https://github.com/ESMCI/mpi-serial + fxurl = https://github.com/ESMCI/mpi-serial fxtag = MPIserial_2.5.0 fxrequired = T:T [submodule "cpl7"] path = components/cpl7 url = https://github.com/ESCOMP/CESM_CPL7andDataComps + fxurl = https://github.com/ESCOMP/CESM_CPL7andDataComps fxtag = cpl77.0.7 fxrequired = T:T [submodule "cam"] path = components/cam url = https://www.github.com/jedwards4b/CAM + fxurl = https://www.github.com/ESCOMP/CAM fxtag = cam6_3_143_rme01 fxrequired = T:T [submodule "ww3"] path = components/ww3 url = https://github.com/ESCOMP/WW3-CESM + fxurl = https://github.com/ESCOMP/WW3-CESM fxtag = ww3_221108 fxrequired = T:T [submodule "ww3dev"] path = components/ww3dev url = https://github.com/jedwards4b/WW3_interface + fxurl = https://github.com/ESCOMP/WW3_interface fxtag = main_0.0.13_rme01 fxrequired = T:T [submodule "rtm"] path = components/rtm url = https://github.com/ESCOMP/RTM + fxurl = https://github.com/ESCOMP/RTM fxrequired = T:T fxtag = rtm1_0_78 [submodule "pysect"] path = tools/statistical_ensemble_test/pyCECT url = https://github.com/NCAR/PyCECT + fxurl = https://github.com/NCAR/PyCECT fxrequired = T:T fxtag = 3.2.2 [submodule "mosart"] path = components/mosart url = https://github.com/ESCOMP/MOSART + fxurl = https://github.com/ESCOMP/MOSART fxrequired = T:T fxtag = mosart1_0_48 [submodule "mizuroute"] path = components/mizuroute url = https://github.com/ESCOMP/mizuRoute + fxurl = https://github.com/ESCOMP/mizuRoute fxrequired = T:T fxtag = cesm-coupling.n02_v2.1.2 [submodule "fms"] path = libraries/FMS url = https://github.com/jedwards4b/FMS_interface + fxurl = https://github.com/ESCOMP/FMS_interface fxrequired = T:T fxtag = fi_230818_rme01 [submodule "parallelio"] path = libraries/parallelio url = https://github.com/NCAR/ParallelIO + fxurl = https://github.com/NCAR/ParallelIO fxrequired = T:T fxtag = pio2_6_2 [submodule "cdeps"] path = components/cdeps url = https://github.com/jedwards4b/CDEPS + fxurl = https://github.com/ESCOMP/CDEPS fxrequired = T:T fxtag = cdeps1.0.26_rme01 [submodule "cmeps"] path = components/cmeps url = https://github.com/ESCOMP/CMEPS.git + fxurl = https://github.com/ESCOMP/CMEPS.git fxrequired = T:T fxtag = cmeps0.14.49 [submodule "cice5"] path = components/cice5 url = https://github.com/ESCOMP/CESM_CICE5 + fxurl = https://github.com/ESCOMP/CESM_CICE5 fxrequired = T:T - fxtag = cice5_20220128 + fxtag = cice5_20220204 [submodule "cice6"] path = components/cice url = https://github.com/jedwards4b/CESM_CICE + fxurl = https://github.com/ESCOMP/CESM_CICE fxrequired = T:T fxtag = cesm_cice6_4_1_10_rme01 [submodule "cism"] path = components/cism url = https://github.com/jedwards4b/cism-wrapper.git + fxurl = https://github.com/ESCOMP/cism-wrapper.git fxtag = cismwrap_2_1_96_rme02 fxrequired = T:T [submodule "clm"] path = components/clm url = https://github.com/jedwards4b/CTSM + fxurl = https://github.com/ESCOMP/CTSM fxrequired = T:T fxtag = branch_tags/CESM3_dev.n04_ctsm5.1.dev145_rme02 [submodule "mom"] path = components/mom url = https://github.com/jedwards4b/MOM_interface + fxurl = https://github.com/ESCOMP/MOM_interface fxrequired = T:T fxtag = mi_231107_rme01 [submodule "pop"] path = components/pop url = https://github.com/jedwards4b/POP2-CESM + fxurl = https://github.com/ESCOMP/POP2-CESM fxrequired = T:T fxtag = cesm_pop_2_1_20231208_rme01 diff --git a/bin/git-fleximod b/bin/git-fleximod index 5add2e54d..c58c7bf14 100755 --- a/bin/git-fleximod +++ b/bin/git-fleximod @@ -26,7 +26,7 @@ def commandline_arguments(args=None): # # user options # - choices = ["update", "install", "status"] + choices = ["update", "install", "status", "test"] parser.add_argument( "action", choices=choices, @@ -110,21 +110,6 @@ def commandline_arguments(args=None): "information to the screen and log file.", ) - logging_group = parser.add_mutually_exclusive_group() - - logging_group.add_argument( - "--logging", - dest="do_logging", - action="store_true", - help="DEVELOPER: enable logging.", - ) - logging_group.add_argument( - "--no-logging", - dest="do_logging", - action="store_false", - default=False, - help="DEVELOPER: disable logging " "(this is the default)", - ) if args: options = parser.parse_args(args) else: @@ -269,6 +254,7 @@ def submodule_checkout(root, name, path, url=None, tag=None): def submodules_status(gitmodules, root_dir): + testfails = 0 for name in gitmodules.sections(): path = gitmodules.get(name, "path") tag = gitmodules.get(name, "fxtag") @@ -291,23 +277,27 @@ def submodules_status(gitmodules, root_dir): print(f"Submodule {name} not checked out, aligned at tag {tag}") else: print(f"Submodule {name} not checked out, out of sync at tag {atag}, expected tag is {tag}") - + testfails += 1 else: with utils.pushd(newpath): git = GitInterface(newpath, logger) atag = git.git_operation("describe", "--tags", "--always").rstrip() if tag and atag != tag: print(f"Submodule {name} {atag} is out of sync with .gitmodules {tag}") + testfails += 1 elif tag: print(f"Submodule {name} at tag {tag}") else: print( f"Submodule {name} has no tag defined in .gitmodules, module at {atag}" ) + testfails += 1 + status = git.git_operation("status","--ignore-submodules","untracked") if "nothing to commit" not in status: print(status) - + + return testfails def submodules_update(gitmodules, root_dir): for name in gitmodules.sections(): @@ -375,7 +365,26 @@ def submodules_install(gitmodules, root_dir, requiredlist): submodule_checkout(root_dir, name, path, url=url, tag=fxtag) +def submodules_test(gitmodules, root_dir): + # First check that fxtags are present and in sync with submodule hashes + testfails = submodules_status(gitmodules, root_dir) + # Then make sure that urls are consistant with fxurls (not forks and not ssh) + # and that sparse checkout files exist + for name in gitmodules.sections(): + url = gitmodules.get(name, "url") + fxurl = gitmodules.get(name, "fxurl") + fxsparse = gitmodules.get(name, "fxsparse") + path = gitmodules.get(name, "path") + if not fxurl or url != fxurl: + print(f"submodule {name} url {url} not in sync with required {fxurl}") + testfails += 1 + if fxsparse and not os.path.isfile(os.path.join(root_dir, path, fxsparse)): + print(f"sparse submodule {name} sparse checkout file {fxsparse} not found") + testfails += 1 + return testfails + + def _main_func(): ( root_dir, @@ -408,16 +417,18 @@ def _main_func(): includelist=includelist, excludelist=excludelist, ) - + retval = 0 if action == "update": submodules_update(gitmodules, root_dir) elif action == "install": submodules_install(gitmodules, root_dir, fxrequired) elif action == "status": submodules_status(gitmodules, root_dir) + elif action == "test": + retval = submodules_test(gitmodules, root_dir) else: utils.fatal_error(f"unrecognized action request {action}") - - + return(retval) + if __name__ == "__main__": - _main_func() + sys.exit(_main_func()) diff --git a/lib/python/site-packages/fleximod/version.py b/lib/python/site-packages/fleximod/version.py index c3bb2961b..1c98a23a8 100644 --- a/lib/python/site-packages/fleximod/version.py +++ b/lib/python/site-packages/fleximod/version.py @@ -1 +1 @@ -__version__ = '0.1.8' +__version__ = '0.1.9'