From 2b0a01e372bf2b8393aaf164985d7d4f46151bc9 Mon Sep 17 00:00:00 2001 From: Weiyuan Jiang Date: Thu, 25 Jan 2024 15:30:25 -0500 Subject: [PATCH 01/12] no remap upper if there is no change --- pre/remap_restart/remap_upper.py | 62 +++++++++++++++++++++++++++----- 1 file changed, 53 insertions(+), 9 deletions(-) diff --git a/pre/remap_restart/remap_upper.py b/pre/remap_restart/remap_upper.py index 1f28c8a..369eca5 100755 --- a/pre/remap_restart/remap_upper.py +++ b/pre/remap_restart/remap_upper.py @@ -17,6 +17,7 @@ from remap_base import remap_base from remap_utils import * from remap_bin2nc import bin2nc +import netCDF4 as nc class upperair(remap_base): def __init__(self, **configs): @@ -59,6 +60,20 @@ def remap(self): out_dir = config['output']['shared']['out_dir'] if not os.path.exists(out_dir) : os.makedirs(out_dir) + + types = '.bin' + type_str = subprocess.check_output(['file','-b', os.path.realpath(restarts_in[0])]) + type_str = str(type_str) + if type_str.find('Hierarchical') >=0: + types = '.nc4' + yyyymmddhh_ = str(config['input']['shared']['yyyymmddhh']) + + label = get_label(config) + suffix = yyyymmddhh_[0:8]+'_'+yyyymmddhh_[8:10] +'z' + types + label + + no_remap = self.link_without_remap(restarts_in, suffix) + if (no_remap) : return + print( "cd " + out_dir) os.chdir(out_dir) @@ -71,15 +86,6 @@ def remap(self): os.chdir(tmpdir) print('\nUpper air restart file names link from "_rst" to "_restart_in" \n') - types = '.bin' - type_str = subprocess.check_output(['file','-b', os.path.realpath(restarts_in[0])]) - type_str = str(type_str) - if type_str.find('Hierarchical') >=0: - types = '.nc4' - yyyymmddhh_ = str(config['input']['shared']['yyyymmddhh']) - - label = get_label(config) - suffix = yyyymmddhh_[0:8]+'_'+yyyymmddhh_[8:10] +'z' + types + label for rst in restarts_in : f = os.path.basename(rst).split('_rst')[0].split('.')[-1]+'_restart_in' cmd = '/bin/ln -s ' + rst + ' ' + f @@ -371,6 +377,7 @@ def copy_merra2(self): merra_2_rst_dir + expid+'.gocart_internal_rst.' + suffix, merra_2_rst_dir + expid+'.pchem_internal_rst.' + suffix, merra_2_rst_dir + expid+'.agcm_import_rst.' + suffix ] + bin2nc_yaml = ['bin2nc_merra2_fv.yaml', 'bin2nc_merra2_moist.yaml', 'bin2nc_merra2_gocart.yaml', 'bin2nc_merra2_pchem.yaml','bin2nc_merra2_agcm.yaml'] bin_path = os.path.dirname(os.path.realpath(__file__)) for (f, yf) in zip(upperin,bin2nc_yaml) : @@ -387,6 +394,43 @@ def copy_merra2(self): bin2nc(dest, ncdest, yaml_file) os.remove(dest) + def link_without_remap(self, restarts_in, suffix): + config = self.config + in_agrid = config['input']['shared']['agrid'] + out_agrid = config['output']['shared']['agrid'] + out_levels = config['output']['air']['nlevel'] + in_bc_version = config['input']['shared']['bc_version'] + out_bc_version = config['output']['shared']['bc_version'] + in_stretch = config['input']['shared']['stretch'] + out_stretch = config['output']['shared']['stretch'] + + if (in_agrid == out_agrid and \ + in_bc_version == out_bc_version and in_stretch == out_stretch) : + + for rst in restarts_in : + if 'fvcore_internal' in rst: + fvrst = nc.Dataset(rst) + in_levels = fvrst.dimensions['lev'].size + if in_levels != int(out_levels): return False + + out_dir = config['output']['shared']['out_dir'] + expid = config['output']['shared']['expid'] + if (expid) : + expid = expid + '.' + else: + expid = '' + + print('\nUpper air restart files link to orignal restart files without remapping" \n') + for rst in restarts_in : + f = expid + os.path.basename(rst).split('_rst')[0].split('.')[-1]+'_rst.'+suffix + cmd = '/bin/ln -s ' + rst + ' ' + out_dir+'/'+f + print('\n'+cmd) + subprocess.call(shlex.split(cmd)) + + return True + + return False + if __name__ == '__main__' : air = upperair(params_file='remap_params.yaml') air.remap() From a01e45d72531ef382343148046600b6c4c1e33c3 Mon Sep 17 00:00:00 2001 From: Weiyuan Jiang Date: Fri, 26 Jan 2024 10:32:54 -0500 Subject: [PATCH 02/12] change log --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index daa19ef..7effe53 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Do not remap (link instead) upper air restarts when it is not necessary - Added EASE grid option for remapping of land restarts in remap_restarts.py package (facilitates use of package in GEOSldas setup script) - Added support for SLES15, NAS site and log for remap_lake_landice_saltwater in remap_restarts.py package - Added "land_only" option for remapping of restarts From 21acf693344be144847c9c6ff4e4a9d23ea95f13 Mon Sep 17 00:00:00 2001 From: Rolf Reichle <54944691+gmao-rreichle@users.noreply.github.com> Date: Fri, 26 Jan 2024 13:32:00 -0500 Subject: [PATCH 03/12] minimal edit of print statement (remap_upper.py) --- pre/remap_restart/remap_upper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pre/remap_restart/remap_upper.py b/pre/remap_restart/remap_upper.py index 369eca5..484abf4 100755 --- a/pre/remap_restart/remap_upper.py +++ b/pre/remap_restart/remap_upper.py @@ -420,7 +420,7 @@ def link_without_remap(self, restarts_in, suffix): else: expid = '' - print('\nUpper air restart files link to orignal restart files without remapping" \n') + print('\nLinking upper air restart files to orignal restart files without remapping\n') for rst in restarts_in : f = expid + os.path.basename(rst).split('_rst')[0].split('.')[-1]+'_rst.'+suffix cmd = '/bin/ln -s ' + rst + ' ' + out_dir+'/'+f From 96343aa5704e450a7f567cebf1208dd514acd7cd Mon Sep 17 00:00:00 2001 From: Weiyuan Jiang Date: Mon, 29 Jan 2024 17:17:25 -0500 Subject: [PATCH 04/12] copy without remapping for upper air and land --- pre/remap_restart/remap_base.py | 41 ++++++++ pre/remap_restart/remap_catchANDcn.py | 24 +++-- .../remap_lake_landice_saltwater.py | 61 ++++++------ pre/remap_restart/remap_upper.py | 96 ++++++------------- 4 files changed, 117 insertions(+), 105 deletions(-) diff --git a/pre/remap_restart/remap_base.py b/pre/remap_restart/remap_base.py index f9ee881..16930bb 100755 --- a/pre/remap_restart/remap_base.py +++ b/pre/remap_restart/remap_base.py @@ -33,3 +33,44 @@ def remove_merra2(self): if self.config['input']['shared']['MERRA-2']: print(" remove temporary folder that contains MERRA-2 archived files ... \n") subprocess.call(['/bin/rm', '-rf', self.config['input']['shared']['rst_dir']]) + + def copy_without_remap(self, restarts_in, compared_file_in, compared_file_out, suffix): + config = self.config + in_agrid = config['input']['shared']['agrid'] + out_agrid = config['output']['shared']['agrid'] + out_levels = config['output']['air']['nlevel'] + in_bc_version = config['input']['shared']['bc_version'] + out_bc_version = config['output']['shared']['bc_version'] + in_stretch = config['input']['shared']['stretch'] + out_stretch = config['output']['shared']['stretch'] + + if (in_agrid == out_agrid and in_stretch == out_stretch): + + for rst in restarts_in : + if 'fvcore_internal' in rst: + fvrst = nc.Dataset(rst) + in_levels = fvrst.dimensions['lev'].size + if in_levels != int(out_levels): return False + + cmd = 'diff -q ' + compared_file_in + ' ' + compared_file_out + print('\n' + cmd) + diff_str = str(subprocess.check_output(shlex.split(cmd))) + if ' differ' in diff_str: return False + + expid = config['output']['shared']['expid'] + if (expid) : + expid = expid + '.' + else: + expid = '' + out_dir = config['output']['shared']['out_dir'] + + print('\nCopy restart files from orignal restart files without remapping. \n') + for rst in restarts_in : + f = expid + os.path.basename(rst).split('_rst')[0].split('.')[-1]+'_rst.'+suffix + cmd = '/bin/cp ' + rst + ' ' + out_dir+'/'+f + print('\n'+cmd) + subprocess.call(shlex.split(cmd)) + + return True + + return False diff --git a/pre/remap_restart/remap_catchANDcn.py b/pre/remap_restart/remap_catchANDcn.py index 59af862..8cdccf7 100755 --- a/pre/remap_restart/remap_catchANDcn.py +++ b/pre/remap_restart/remap_catchANDcn.py @@ -43,7 +43,6 @@ def remap(self): return in_rstfile = in_rstfiles[0] - print("\nRemapping " + model + ".....\n") cwdir = os.getcwd() bindir = os.path.dirname(os.path.realpath(__file__)) @@ -84,6 +83,19 @@ def remap(self): out_bc_landdir = get_landdir(out_bc_base, out_bc_version, agrid=agrid, ogrid=ogrid, omodel=omodel, stretch=stretch, grid=EASE_grid) + label = get_label(config) + + suffix = time+'z.nc4' + label + + if (expid) : + expid = expid + '.' + else: + expid = '' + + no_remap = self.copy_without_remap([in_rstfile], in_tilefile, out_tilefile, suffix) + if (no_remap) : return + + print("\nRemapping " + model + ".....\n") # determine NPE based on *approximate* number of input and output tile in_Ntile = 0 @@ -131,14 +143,6 @@ def remap(self): account = config['slurm_pbs']['account'] # even if the (MERRA-2) input restarts are binary, the output restarts will always be nc4 (remap_bin2nc.py) - label = get_label(config) - - suffix = time+'z.nc4' + label - - if (expid) : - expid = expid + '.' - else: - expid = '' suffix = '_rst.' + suffix out_rstfile = expid + os.path.basename(in_rstfile).split('_rst')[0].split('.')[-1]+suffix @@ -433,6 +437,8 @@ def remap_land_only(): catch = catchANDcn(params_file=config_yaml) catch.remap() + + if __name__ == '__main__' : remap_land_only() diff --git a/pre/remap_restart/remap_lake_landice_saltwater.py b/pre/remap_restart/remap_lake_landice_saltwater.py index 5603e1c..ca2f9cd 100755 --- a/pre/remap_restart/remap_lake_landice_saltwater.py +++ b/pre/remap_restart/remap_lake_landice_saltwater.py @@ -37,12 +37,45 @@ def remap(self): config = self.config cwdir = os.getcwd() bindir = os.path.dirname(os.path.realpath(__file__)) + in_bc_base = config['input']['shared']['bc_base'] in_bc_version = config['input']['shared']['bc_version'] out_bc_base = config['output']['shared']['bc_base'] out_bc_version= config['output']['shared']['bc_version'] + agrid = config['input']['shared']['agrid'] + ogrid = config['input']['shared']['ogrid'] + omodel = config['input']['shared']['omodel'] + stretch = config['input']['shared']['stretch'] + in_geomdir = get_geomdir(in_bc_base, in_bc_version, agrid=agrid, ogrid=ogrid, omodel=omodel, stretch=stretch) + in_tile_file = glob.glob(in_geomdir+ '/*-Pfafstetter.til')[0] + + agrid = config['output']['shared']['agrid'] + ogrid = config['output']['shared']['ogrid'] + omodel = config['output']['shared']['omodel'] + stretch = config['output']['shared']['stretch'] + out_geomdir = get_geomdir(out_bc_base, out_bc_version, agrid=agrid, ogrid=ogrid, omodel=omodel, stretch=stretch) + out_tile_file = glob.glob(out_geomdir+ '/*-Pfafstetter.til')[0] + + types = '.bin' + type_str = sp.check_output(['file','-b', os.path.realpath(restarts_in[0])]) + type_str = str(type_str) + if 'Hierarchical' in type_str: + types = '.nc4' + yyyymmddhh_ = str(config['input']['shared']['yyyymmddhh']) + + label = get_label(config) + suffix = yyyymmddhh_[0:8]+'_'+yyyymmddhh_[8:10] +'z' + types + label + out_dir = config['output']['shared']['out_dir'] + expid = config['output']['shared']['expid'] + if (expid) : + expid = expid + '.' + else: + expid = '' + + no_remap = self.copy_without_remap(restarts_in, in_tile_file, out_tile_file, suffix) + if no_remap : return if not os.path.exists(out_dir) : os.makedirs(out_dir) print( "cd " + out_dir) @@ -58,15 +91,6 @@ def remap(self): print ("mkdir " + OutData_dir) os.makedirs(OutData_dir) - types = '.bin' - type_str = sp.check_output(['file','-b', os.path.realpath(restarts_in[0])]) - type_str = str(type_str) - if 'Hierarchical' in type_str: - types = '.nc4' - yyyymmddhh_ = str(config['input']['shared']['yyyymmddhh']) - - label = get_label(config) - suffix = yyyymmddhh_[0:8]+'_'+yyyymmddhh_[8:10] +'z' + types + label saltwater = '' seaice = '' @@ -87,20 +111,6 @@ def remap(self): if 'roue' in f : route = f if 'openwater' in f : openwater = f - agrid = config['input']['shared']['agrid'] - ogrid = config['input']['shared']['ogrid'] - omodel = config['input']['shared']['omodel'] - stretch = config['input']['shared']['stretch'] - in_geomdir = get_geomdir(in_bc_base, in_bc_version, agrid=agrid, ogrid=ogrid, omodel=omodel, stretch=stretch) - in_tile_file = glob.glob(in_geomdir+ '/*-Pfafstetter.til')[0] - - agrid = config['output']['shared']['agrid'] - ogrid = config['output']['shared']['ogrid'] - omodel = config['output']['shared']['omodel'] - stretch = config['output']['shared']['stretch'] - out_geomdir = get_geomdir(out_bc_base, out_bc_version, agrid=agrid, ogrid=ogrid, omodel=omodel, stretch=stretch) - out_tile_file = glob.glob(out_geomdir+ '/*-Pfafstetter.til')[0] - in_til = InData_dir+'/' + os.path.basename(in_tile_file) out_til = OutData_dir+'/'+ os.path.basename(out_tile_file) @@ -152,11 +162,6 @@ def remap(self): cmd = route + out_til + ' ' + yyyymmddhh_[0:6] self.run_and_log(cmd, log_name) - expid = config['output']['shared']['expid'] - if (expid) : - expid = expid + '.' - else: - expid = '' suffix = '_rst.' + suffix for out_rst in glob.glob("OutData/*_rst*"): filename = expid + os.path.basename(out_rst).split('_rst')[0].split('.')[-1]+suffix diff --git a/pre/remap_restart/remap_upper.py b/pre/remap_restart/remap_upper.py index 369eca5..6704d8d 100755 --- a/pre/remap_restart/remap_upper.py +++ b/pre/remap_restart/remap_upper.py @@ -71,7 +71,34 @@ def remap(self): label = get_label(config) suffix = yyyymmddhh_[0:8]+'_'+yyyymmddhh_[8:10] +'z' + types + label - no_remap = self.link_without_remap(restarts_in, suffix) + in_bc_base = config['input']['shared']['bc_base'] + if "gmao_SIteam/ModelData" in in_bc_base: + assert GEOS_SITE == "NAS", "wrong site to run the package" + + in_bc_version = config['input']['shared']['bc_version'] + agrid = config['input']['shared']['agrid'] + ogrid = config['input']['shared']['ogrid'] + omodel = config['input']['shared']['omodel'] + stretch = config['input']['shared']['stretch'] + topo_bcsdir = get_topodir(in_bc_base, in_bc_version, agrid=agrid, ogrid=ogrid, omodel=omodel, stretch=stretch) + topoin = glob.glob(topo_bcsdir+'/topo_DYN_ave*.data')[0] + + out_bc_base = config['output']['shared']['bc_base'] + out_bc_version = config['output']['shared']['bc_version'] + agrid = config['output']['shared']['agrid'] + ogrid = config['output']['shared']['ogrid'] + omodel = config['output']['shared']['omodel'] + stretch = config['output']['shared']['stretch'] + topo_bcsdir = get_topodir(out_bc_base, out_bc_version, agrid=agrid, ogrid=ogrid, omodel=omodel, stretch=stretch) + topoout = glob.glob(topo_bcsdir+'/topo_DYN_ave*.data')[0] + + expid = config['output']['shared']['expid'] + if (expid) : + expid = expid + '.' + else: + expid = '' + + no_remap = self.copy_without_remap(restarts_in, topoin, topoout, suffix) if (no_remap) : return print( "cd " + out_dir) @@ -92,40 +119,15 @@ def remap(self): print('\n'+cmd) subprocess.call(shlex.split(cmd)) - in_bc_base = config['input']['shared']['bc_base'] - in_bc_version = config['input']['shared']['bc_version'] - agrid = config['input']['shared']['agrid'] - ogrid = config['input']['shared']['ogrid'] - omodel = config['input']['shared']['omodel'] - stretch = config['input']['shared']['stretch'] - topo_bcsdir = get_topodir(in_bc_base, in_bc_version, agrid=agrid, ogrid=ogrid, omodel=omodel, stretch=stretch) - - if "gmao_SIteam/ModelData" in in_bc_base: - assert GEOS_SITE == "NAS", "wrong site to run the package" - - topoin = glob.glob(topo_bcsdir+'/topo_DYN_ave*.data')[0] # link topo file cmd = '/bin/ln -s ' + topoin + ' .' print('\n'+cmd) subprocess.call(shlex.split(cmd)) - out_bc_base = config['output']['shared']['bc_base'] - out_bc_version = config['output']['shared']['bc_version'] - agrid = config['output']['shared']['agrid'] - ogrid = config['output']['shared']['ogrid'] - omodel = config['output']['shared']['omodel'] - stretch = config['output']['shared']['stretch'] - topo_bcsdir = get_topodir(out_bc_base, out_bc_version, agrid=agrid, ogrid=ogrid, omodel=omodel, stretch=stretch) - - topoout = glob.glob(topo_bcsdir+'/topo_DYN_ave*.data')[0] cmd = '/bin/ln -s ' + topoout + ' topo_dynave.data' print('\n'+cmd) subprocess.call(shlex.split(cmd)) - #fname = os.path.basename(topoout) - #cmd = '/bin/ln -s ' + fname + ' topo_dynave.data' - #print('\n'+cmd) - #subprocess.call(shlex.split(cmd)) agrid = config['output']['shared']['agrid'] if agrid[0].upper() == 'C': @@ -313,11 +315,6 @@ def remap(self): # # post process # - expid = config['output']['shared']['expid'] - if (expid) : - expid = expid + '.' - else: - expid = '' suffix = '_rst.' + suffix for out_rst in glob.glob("*_rst*"): @@ -394,43 +391,6 @@ def copy_merra2(self): bin2nc(dest, ncdest, yaml_file) os.remove(dest) - def link_without_remap(self, restarts_in, suffix): - config = self.config - in_agrid = config['input']['shared']['agrid'] - out_agrid = config['output']['shared']['agrid'] - out_levels = config['output']['air']['nlevel'] - in_bc_version = config['input']['shared']['bc_version'] - out_bc_version = config['output']['shared']['bc_version'] - in_stretch = config['input']['shared']['stretch'] - out_stretch = config['output']['shared']['stretch'] - - if (in_agrid == out_agrid and \ - in_bc_version == out_bc_version and in_stretch == out_stretch) : - - for rst in restarts_in : - if 'fvcore_internal' in rst: - fvrst = nc.Dataset(rst) - in_levels = fvrst.dimensions['lev'].size - if in_levels != int(out_levels): return False - - out_dir = config['output']['shared']['out_dir'] - expid = config['output']['shared']['expid'] - if (expid) : - expid = expid + '.' - else: - expid = '' - - print('\nUpper air restart files link to orignal restart files without remapping" \n') - for rst in restarts_in : - f = expid + os.path.basename(rst).split('_rst')[0].split('.')[-1]+'_rst.'+suffix - cmd = '/bin/ln -s ' + rst + ' ' + out_dir+'/'+f - print('\n'+cmd) - subprocess.call(shlex.split(cmd)) - - return True - - return False - if __name__ == '__main__' : air = upperair(params_file='remap_params.yaml') air.remap() From 2b2d6d772acabad9a44f25f6c8d6937cea6f3b78 Mon Sep 17 00:00:00 2001 From: Weiyuan Jiang Date: Thu, 1 Feb 2024 09:21:27 -0500 Subject: [PATCH 05/12] use return code for diff --- pre/remap_restart/remap_base.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pre/remap_restart/remap_base.py b/pre/remap_restart/remap_base.py index 16930bb..53dcc7c 100755 --- a/pre/remap_restart/remap_base.py +++ b/pre/remap_restart/remap_base.py @@ -54,8 +54,9 @@ def copy_without_remap(self, restarts_in, compared_file_in, compared_file_out, s cmd = 'diff -q ' + compared_file_in + ' ' + compared_file_out print('\n' + cmd) - diff_str = str(subprocess.check_output(shlex.split(cmd))) - if ' differ' in diff_str: return False + diff = subprocess.call(shlex.split(cmd))) + # diff = 0 means no difference + if diff != 0: return False expid = config['output']['shared']['expid'] if (expid) : From 31f8444463f4fd0095c47038d1f08ded6dcfd1f2 Mon Sep 17 00:00:00 2001 From: Biljana Orescanin <68251545+biljanaorescanin@users.noreply.github.com> Date: Thu, 1 Feb 2024 09:40:34 -0500 Subject: [PATCH 06/12] typo --- pre/remap_restart/remap_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pre/remap_restart/remap_base.py b/pre/remap_restart/remap_base.py index 53dcc7c..7aae23e 100755 --- a/pre/remap_restart/remap_base.py +++ b/pre/remap_restart/remap_base.py @@ -54,7 +54,7 @@ def copy_without_remap(self, restarts_in, compared_file_in, compared_file_out, s cmd = 'diff -q ' + compared_file_in + ' ' + compared_file_out print('\n' + cmd) - diff = subprocess.call(shlex.split(cmd))) + diff = subprocess.call(shlex.split(cmd)) # diff = 0 means no difference if diff != 0: return False From 18588773f2ef58e92cdb3b473d92a3cd3457a53b Mon Sep 17 00:00:00 2001 From: Weiyuan Jiang Date: Thu, 1 Feb 2024 09:56:28 -0500 Subject: [PATCH 07/12] add more condition for catchment restart comparison --- pre/remap_restart/remap_base.py | 6 +++++- pre/remap_restart/remap_catchANDcn.py | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/pre/remap_restart/remap_base.py b/pre/remap_restart/remap_base.py index 53dcc7c..0d85e70 100755 --- a/pre/remap_restart/remap_base.py +++ b/pre/remap_restart/remap_base.py @@ -34,7 +34,7 @@ def remove_merra2(self): print(" remove temporary folder that contains MERRA-2 archived files ... \n") subprocess.call(['/bin/rm', '-rf', self.config['input']['shared']['rst_dir']]) - def copy_without_remap(self, restarts_in, compared_file_in, compared_file_out, suffix): + def copy_without_remap(self, restarts_in, compared_file_in, compared_file_out, suffix, catch=False): config = self.config in_agrid = config['input']['shared']['agrid'] out_agrid = config['output']['shared']['agrid'] @@ -58,6 +58,10 @@ def copy_without_remap(self, restarts_in, compared_file_in, compared_file_out, s # diff = 0 means no difference if diff != 0: return False + # for catchment, even tile files are the same, if bc is different, it still need remap + if (catch) : + if in_bc_version != out_bc_version : return False + expid = config['output']['shared']['expid'] if (expid) : expid = expid + '.' diff --git a/pre/remap_restart/remap_catchANDcn.py b/pre/remap_restart/remap_catchANDcn.py index 8cdccf7..8481707 100755 --- a/pre/remap_restart/remap_catchANDcn.py +++ b/pre/remap_restart/remap_catchANDcn.py @@ -92,7 +92,7 @@ def remap(self): else: expid = '' - no_remap = self.copy_without_remap([in_rstfile], in_tilefile, out_tilefile, suffix) + no_remap = self.copy_without_remap([in_rstfile], in_tilefile, out_tilefile, suffix, catch=True) if (no_remap) : return print("\nRemapping " + model + ".....\n") From a6e4ea8aa60cac4cee66e75ece6e77990917e1b2 Mon Sep 17 00:00:00 2001 From: Rolf Reichle Date: Fri, 2 Feb 2024 09:51:39 -0500 Subject: [PATCH 08/12] added documentation to function copy_without_remap (remap_base.py) --- pre/remap_restart/remap_base.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/pre/remap_restart/remap_base.py b/pre/remap_restart/remap_base.py index bfd2a10..856582e 100755 --- a/pre/remap_restart/remap_base.py +++ b/pre/remap_restart/remap_base.py @@ -35,6 +35,16 @@ def remove_merra2(self): subprocess.call(['/bin/rm', '-rf', self.config['input']['shared']['rst_dir']]) def copy_without_remap(self, restarts_in, compared_file_in, compared_file_out, suffix, catch=False): +# +# Determine if remapping is needed for a group of restart files, or if the input restart files +# can just simply be copied to the output dir, based on the following dependency table: +# +# restarts | agrid/stretch | #levels | topo files | tile file | bcs version +# ---------------------------------------------------------------------------- +# upper air | X | X | X | | +# catch/vegdyn | X | | | X | X +# landice | X | | | X | +# config = self.config in_agrid = config['input']['shared']['agrid'] out_agrid = config['output']['shared']['agrid'] From e6106b472257e5c810131ab3c87146e2f2126826 Mon Sep 17 00:00:00 2001 From: Rolf Reichle Date: Fri, 2 Feb 2024 12:30:39 -0500 Subject: [PATCH 09/12] minimal white-space change to fix alignment of table in comment (remap_base.py) --- pre/remap_restart/remap_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pre/remap_restart/remap_base.py b/pre/remap_restart/remap_base.py index 856582e..d018911 100755 --- a/pre/remap_restart/remap_base.py +++ b/pre/remap_restart/remap_base.py @@ -42,7 +42,7 @@ def copy_without_remap(self, restarts_in, compared_file_in, compared_file_out, s # restarts | agrid/stretch | #levels | topo files | tile file | bcs version # ---------------------------------------------------------------------------- # upper air | X | X | X | | -# catch/vegdyn | X | | | X | X +# catch/vegdyn | X | | | X | X # landice | X | | | X | # config = self.config From 1d523559018f839c484eea5801b22858b760b11a Mon Sep 17 00:00:00 2001 From: Rolf Reichle Date: Fri, 2 Feb 2024 12:33:51 -0500 Subject: [PATCH 10/12] updated CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b426ddc..2fa812e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added -- Do not remap (link instead) upper air restarts when it is not necessary +- Do not remap groups of restarts when it is not necessary, make copies instead. - Addition of SST and FRACI forecast (ocean) boundary conditions generation capability in `pre/prepare_ocnExtData` - Added EASE grid option for remapping of land restarts in remap_restarts.py package (facilitates use of package in GEOSldas setup script) - Added support for SLES15, NAS site and log for remap_lake_landice_saltwater in remap_restarts.py package From aa12c3c97343496b30b885b5e42e553f7dfa986c Mon Sep 17 00:00:00 2001 From: Rolf Reichle Date: Wed, 7 Feb 2024 13:17:56 -0500 Subject: [PATCH 11/12] clarified USAGE of remap_restarts.py (0-diff-trivial) --- pre/remap_restart/remap_restarts.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pre/remap_restart/remap_restarts.py b/pre/remap_restart/remap_restarts.py index 78ae59f..560b6c9 100755 --- a/pre/remap_restart/remap_restarts.py +++ b/pre/remap_restart/remap_restarts.py @@ -37,6 +37,9 @@ 2. Use an existing yaml config file: ./remap_restarts.py config_file -c my_config.yaml + NOTE: The yaml interface may not backward compatible across releases. If the + existing yaml file does not work, use the questionary option above to generate + an updated yaml file. 3. Use command line arguments: ./remap_restarts.py command_line -ymdh 2004041421 .... From 5ac1609a6bff8135d07646db3a6a97df58c5ce4a Mon Sep 17 00:00:00 2001 From: Rolf Reichle Date: Wed, 7 Feb 2024 13:55:34 -0500 Subject: [PATCH 12/12] making remap_restarts.py USAGE text consistent with corresponding Wiki page (0-diff-trivial) --- pre/remap_restart/remap_restarts.py | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/pre/remap_restart/remap_restarts.py b/pre/remap_restart/remap_restarts.py index 560b6c9..f2a380f 100755 --- a/pre/remap_restart/remap_restarts.py +++ b/pre/remap_restart/remap_restarts.py @@ -26,27 +26,28 @@ program_description = textwrap.dedent(f''' USAGE: - This script provides three options for remapping GEOS restart files: + This script provides four options for remapping GEOS restart files: - 1. Use the interactive questionary: + 1. Use the interactive questionnaire (recommended): ./remap_restarts.py - The questionary concludes with the option to submit the remapping job. + The questionnaire concludes with the option to submit the remapping job. It also creates a yaml configuration file (`remap_params.yaml`) and - a command line options string (`remap_restarts.CMD`), which can be edited - manually and used in the other two ways of running `remap_restarts.py`. + a matching command line argument string (`remap_restarts.CMD`), which can be edited + manually and used in the next two run options. - 2. Use an existing yaml config file: + 2. Use an existing yaml config file*: ./remap_restarts.py config_file -c my_config.yaml - NOTE: The yaml interface may not backward compatible across releases. If the - existing yaml file does not work, use the questionary option above to generate - an updated yaml file. - 3. Use command line arguments: - ./remap_restarts.py command_line -ymdh 2004041421 .... + 3. Use command line arguments*: + ./remap_restarts.py command_line -ymdh 2004041421 ... 4. For GEOSldas: Remap land (catch[cn]) restart only; global domain only; ens0000 only: ./remap_restarts.py land_only + *NOTE: The yaml and command-line interfaces may not backward compatible across + releases. If existing yaml files or command-line strings do not work, + use the questionnaire option to generate updated versions. + Help commands: ./remap_restarts.py -h ./remap_restarts.py config_file -h