Merge branch 'nuopc_updates'

Various updates needed for NUOPC

Also adds mizuRoute as an external component

Externals.cfg

@@ -26,19 +26,32 @@ repo_url = https://github.com/ESCOMP/MOSART
 tag = mosart1_0_38
 required = True
 
-[cdeps]
-hash = 45b7a85
+[mizuRoute]
+local_path = components/mizuRoute
 protocol = git
-repo_url = https://github.com/ESCOMP/CDEPS.git
-local_path = components/cdeps
+repo_url = https://github.com/nmizukami/mizuRoute
+hash = 34723c2
 required = True
 
 [cime]
 local_path = cime
 protocol = git
 repo_url = https://github.com/ESMCI/cime
-tag = branch_tags/cime5.8.32_a01
-externals = ../Externals_cime.cfg
+tag = branch_tags/cime5.8.32_a02
+required = True
+
+[cmeps]
+local_path = cime/src/drivers/nuopc/
+protocol = git
+repo_url = https://github.com/ESCOMP/CMEPS.git
+hash = 7654038
+required = True
+
+[cdeps]
+local_path = components/cdeps
+protocol = git
+repo_url = https://github.com/ESCOMP/CDEPS.git
+hash = 45b7a85
 required = True
 
 [doc-builder]

cime_config/buildlib

@@ -109,6 +109,7 @@ def _main_func():
             expect( ((compname == "clm") or (compname == "ctsm")), "Unexpected COMP_LND name: %s" % (compname))
 
             paths = [os.path.join(caseroot,"SourceMods","src."+compname),
+                     os.path.join(lnd_root,"src","cpl",driver),
                      os.path.join(lnd_root,"src","main"),
                      os.path.join(lnd_root,"src","biogeophys"),
                      os.path.join(lnd_root,"src","biogeochem"),
@@ -124,8 +125,7 @@ def _main_func():
                      os.path.join(lnd_root,"src","fates","parteh"),
                      os.path.join(lnd_root,"src","utils"),
                      os.path.join(lnd_root,"src","cpl"),
-                     os.path.join(lnd_root,"src","cpl","utils"),
-                     os.path.join(lnd_root,"src","cpl",driver)]
+                     os.path.join(lnd_root,"src","cpl","utils")]
 
             if lilac_mode == 'on':
                 paths.append(os.path.join(lnd_root,"lilac","src"))

cime_config/config_compsets.xml

@@ -65,6 +65,11 @@
     <lname>2000_DATM%GSWP3v1_CLM50%SP_SICE_SOCN_RTM_SGLC_SWAV</lname>
   </compset>
 
+  <compset>
+    <alias>I2000Clm50SpMizGs</alias>
+    <lname>2000_DATM%GSWP3v1_CLM50%SP_SICE_SOCN_MIZUROUTE_SGLC_SWAV</lname>
+  </compset>
+
   <compset>
     <alias>I2010Clm50Sp</alias>
     <lname>2010_DATM%GSWP3v1_CLM50%SP_SICE_SOCN_MOSART_SGLC_SWAV</lname>

doc/ChangeLog

@@ -1,4 +1,106 @@
 ===============================================================
+Tag name:  ctsm5.1.dev016
+Originator(s): mvertens (Mariana Vertenstein), jedwards (Jim Edwards),
+               sacks (Bill Sacks), erik (Erik Kluzek)
+Date:  Tue Nov 17 10:49:40 MST 2020
+One-line Summary: Updates for nuopc
+
+Purpose of changes
+------------------
+
+Various updates needed for NUOPC.
+
+Also adds mizuRoute as an external component.
+
+
+Significant changes to scientifically-supported configurations
+--------------------------------------------------------------
+
+Does this tag change answers significantly for any of the following physics configurations?
+(Details of any changes will be given in the "Answer changes" section below.)
+
+    [Put an [X] in the box for any configuration with significant answer changes.]
+
+[ ] clm5_1
+
+[ ] clm5_0
+
+[ ] ctsm5_0-nwp
+
+[ ] clm4_5
+
+Notes of particular relevance for users
+---------------------------------------
+
+Caveats for users (e.g., need to interpolate initial conditions): none
+
+Changes to CTSM's user interface (e.g., new/renamed XML or namelist variables): none
+
+Changes made to namelist defaults (e.g., changed parameter values): none
+
+Changes to the datasets (e.g., parameter, surface or initial files): none
+
+Substantial timing or memory changes: none
+
+Notes of particular relevance for developers: (including Code reviews and testing)
+---------------------------------------------
+NOTE: Be sure to review the steps in README.CHECKLIST.master_tags as well as the coding style in the Developers Guide
+
+Caveats for developers (e.g., code that is duplicated that requires double maintenance):
+- SoilMoistureStreamMod now is duplicated: the version used with mct is
+  still in src/biogeochem, but there is a new version used with nuopc
+  and the CDEPS data models in src/cpl/nuopc. The latter has just been
+  introduced for testing purposes. Eventually, once we switch everything
+  to use CDEPS, we'll remove this duplicated version.
+
+Changes to tests or testing: none
+
+CTSM testing:
+
+ [PASS means all tests PASS and OK means tests PASS other than expected fails.]
+
+  build-namelist tests:
+
+    cheyenne - not run
+
+  tools-tests (test/tools):
+
+    cheyenne - not run
+
+  PTCLM testing (tools/shared/PTCLM/test):
+
+    cheyenne - not run
+
+  python testing (see instructions in python/README.md; document testing done):
+
+    (any machine) - not run
+
+  regular tests (aux_clm):
+
+    cheyenne ---- pass
+    izumi ------- pass
+
+If the tag used for baseline comparisons was NOT the previous tag, note that here:
+
+
+Answer changes
+--------------
+
+Changes answers relative to baseline: NO
+
+Detailed list of changes
+------------------------
+
+List any externals directories updated (cime, rtm, mosart, cism, fates, etc.):
+- Added mizuRoute (at 34723c2)
+- cime: branch_tags/cime5.8.32_a01 -> branch_tags/cime5.8.32_a02
+- (Some other rearrangement of Externals files, but versions stayed the
+  same)
+
+Pull Requests that document the changes (include PR ids): none
+
+===============================================================
+===============================================================
 Tag name:  ctsm5.1.dev015
 Originator(s):  sacks (Bill Sacks)
 Date:  Fri Nov 13 11:52:34 MST 2020

doc/ChangeSum

@@ -1,5 +1,6 @@
 Tag                   Who      Date  Summary
 ============================================================================================================================
+  ctsm5.1.dev016 multiple 11/17/2020 Updates for nuopc
   ctsm5.1.dev015    sacks 11/13/2020 A few small fixes
   ctsm5.1.dev014 dll/oles 11/07/2020 Grass/crop snow burial fraction fix, and some other fixes
   ctsm5.1.dev013 jedwards 11/04/2020 Allow pnetcdf for vector history files

manage_externals/.gitignore

@@ -12,3 +12,6 @@ components/
 
 # generated python files
 *.pyc
+
+# test tmp file
+test/tmp

manage_externals/.travis.yml

@@ -1,7 +1,6 @@
 language: python
 os: linux
-python: 
-  - "2.7"
+python:
   - "3.4"
   - "3.5"
   - "3.6"

manage_externals/manic/checkout.py

@@ -279,6 +279,9 @@ def commandline_arguments(args=None):
                         help='The externals description filename. '
                         'Default: %(default)s.')
 
+    parser.add_argument('-x', '--exclude', nargs='*',
+                        help='Component(s) listed in the externals file which should be ignored.')
+
     parser.add_argument('-o', '--optional', action='store_true', default=False,
                         help='By default only the required externals '
                         'are checked out. This flag will also checkout the '
@@ -362,7 +365,7 @@ def main(args):
     root_dir = os.path.abspath(os.getcwd())
     external_data = read_externals_description_file(root_dir, args.externals)
     external = create_externals_description(
-        external_data, components=args.components)
+        external_data, components=args.components, exclude=args.exclude)
 
     for comp in args.components:
         if comp not in external.keys():
@@ -377,29 +380,37 @@ def main(args):
 
     if args.status:
         # user requested status-only
-        for comp in sorted(tree_status.keys()):
+        for comp in sorted(tree_status):
             tree_status[comp].log_status_message(args.verbose)
     else:
         # checkout / update the external repositories.
         safe_to_update = check_safe_to_update_repos(tree_status)
         if not safe_to_update:
             # print status
-            for comp in sorted(tree_status.keys()):
+            for comp in sorted(tree_status):
                 tree_status[comp].log_status_message(args.verbose)
             # exit gracefully
             msg = """The external repositories labeled with 'M' above are not in a clean state.
 
-The following are two options for how to proceed:
+The following are three options for how to proceed:
 
-(1) Go into each external that is not in a clean state and issue either
-    an 'svn status' or a 'git status' command. Either revert or commit
-    your changes so that all externals are in a clean state. (Note,
-    though, that it is okay to have untracked files in your working
+(1) Go into each external that is not in a clean state and issue either a 'git status' or
+    an 'svn status' command (depending on whether the external is managed by git or
+    svn). Either revert or commit your changes so that all externals are in a clean
+    state. (To revert changes in git, follow the instructions given when you run 'git
+    status'.) (Note, though, that it is okay to have untracked files in your working
     directory.) Then rerun {program_name}.
 
-(2) Alternatively, you do not have to rely on {program_name}. Instead, you
-    can manually update out-of-sync externals (labeled with 's' above)
-    as described in the configuration file {config_file}.
+(2) Alternatively, you do not have to rely on {program_name}. Instead, you can manually
+    update out-of-sync externals (labeled with 's' above) as described in the
+    configuration file {config_file}. (For example, run 'git fetch' and 'git checkout'
+    commands to checkout the appropriate tags for each external, as given in
+    {config_file}.)
+
+(3) You can also use {program_name} to manage most, but not all externals: You can specify
+    one or more externals to ignore using the '-x' or '--exclude' argument to
+    {program_name}. Excluding externals labeled with 'M' will allow {program_name} to
+    update the other, non-excluded externals.
 
 
 The external repositories labeled with '?' above are not under version

manage_externals/manic/externals_description.py

@@ -253,9 +253,21 @@ def read_gitmodules_file(root_dir, file_name):
                                           ExternalsDescription.REPO_URL, url)
                 externals_description.set(sec_name,
                                           ExternalsDescription.REQUIRED, 'True')
-                git_hash = submods[sec_name]['hash']
-                externals_description.set(sec_name,
-                                          ExternalsDescription.HASH, git_hash)
+                if sec_name in submods:
+                    submod_name = sec_name
+                else:
+                    # The section name does not have to match the path
+                    submod_name = path
+
+                if submod_name in submods:
+                    git_hash = submods[submod_name]['hash']
+                    externals_description.set(sec_name,
+                                              ExternalsDescription.HASH,
+                                              git_hash)
+                else:
+                    emsg = "submodule status has no section, '{}'"
+                    emsg += "\nCheck section names in externals config file"
+                    fatal_error(emsg.format(submod_name))
 
         # Required items
         externals_description.add_section(DESCRIPTION_SECTION)
@@ -264,18 +276,18 @@ def read_gitmodules_file(root_dir, file_name):
     return externals_description
 
 def create_externals_description(
-        model_data, model_format='cfg', components=None, parent_repo=None):
+        model_data, model_format='cfg', components=None, exclude=None, parent_repo=None):
     """Create the a externals description object from the provided data
     """
     externals_description = None
     if model_format == 'dict':
         externals_description = ExternalsDescriptionDict(
-            model_data, components=components)
+            model_data, components=components, exclude=exclude)
     elif model_format == 'cfg':
         major, _, _ = get_cfg_schema_version(model_data)
         if major == 1:
             externals_description = ExternalsDescriptionConfigV1(
-                model_data, components=components, parent_repo=parent_repo)
+                model_data, components=components, exclude=exclude, parent_repo=parent_repo)
         else:
             msg = ('Externals description file has unsupported schema '
                    'version "{0}".'.format(major))
@@ -710,7 +722,7 @@ class ExternalsDescriptionDict(ExternalsDescription):
 
     """
 
-    def __init__(self, model_data, components=None):
+    def __init__(self, model_data, components=None, exclude=None):
         """Parse a native dictionary into a externals description.
         """
         ExternalsDescription.__init__(self)
@@ -722,10 +734,15 @@ def __init__(self, model_data, components=None):
         self._input_patch = 0
         self._verify_schema_version()
         if components:
-            for key in model_data.items():
+            for key in list(model_data.keys()):
                 if key not in components:
                     del model_data[key]
 
+        if exclude:
+            for key in list(model_data.keys()):
+                if key in exclude:
+                    del model_data[key]
+
         self.update(model_data)
         self._check_user_input()
 
@@ -736,7 +753,7 @@ class ExternalsDescriptionConfigV1(ExternalsDescription):
 
     """
 
-    def __init__(self, model_data, components=None, parent_repo=None):
+    def __init__(self, model_data, components=None, exclude=None, parent_repo=None):
         """Convert the config data into a standardized dict that can be used to
         construct the source objects
 
@@ -749,7 +766,7 @@ def __init__(self, model_data, components=None, parent_repo=None):
             get_cfg_schema_version(model_data)
         self._verify_schema_version()
         self._remove_metadata(model_data)
-        self._parse_cfg(model_data, components=components)
+        self._parse_cfg(model_data, components=components, exclude=exclude)
         self._check_user_input()
 
     @staticmethod
@@ -761,7 +778,7 @@ def _remove_metadata(model_data):
         """
         model_data.remove_section(DESCRIPTION_SECTION)
 
-    def _parse_cfg(self, cfg_data, components=None):
+    def _parse_cfg(self, cfg_data, components=None, exclude=None):
         """Parse a config_parser object into a externals description.
         """
         def list_to_dict(input_list, convert_to_lower_case=True):
@@ -778,7 +795,7 @@ def list_to_dict(input_list, convert_to_lower_case=True):
 
         for section in cfg_data.sections():
             name = config_string_cleaner(section.lower().strip())
-            if components and name not in components:
+            if (components and name not in components) or (exclude and name in exclude):
                 continue
             self[name] = {}
             self[name].update(list_to_dict(cfg_data.items(section)))