From 960c11cf50d63e194752341b80d376351583f3b6 Mon Sep 17 00:00:00 2001 From: Aidan Heerdegen Date: Fri, 29 Nov 2019 10:38:19 +1100 Subject: [PATCH 1/5] Add module environment variables to PBS environment when job submitted using payu run --- payu/cli.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/payu/cli.py b/payu/cli.py index 6f60805b..745a2937 100644 --- a/payu/cli.py +++ b/payu/cli.py @@ -119,6 +119,12 @@ def set_env_vars(init_run=None, n_runs=None, lab_path=None, dir_path=None, if reproduce: payu_env_vars['PAYU_REPRODUCE'] = reproduce + # Pass through important module related environment variables + module_env_vars = ['MODULESHOME', 'MODULES_CMD', 'MODULEPATH', 'MODULEV'] + for var in module_env_vars: + if var in os.environ: + payu_env_vars[var] = os.environ[var] + return payu_env_vars From 401309de2f1dc2a41a1992933b7567d59525c82d Mon Sep 17 00:00:00 2001 From: Aidan Heerdegen Date: Fri, 29 Nov 2019 10:39:13 +1100 Subject: [PATCH 2/5] Check to make sure loaded modules are not an empty string. This occurs when LOADED_MODULES is not defined, and defaults to an empty string, which split takes and outputs an array with a single empty string. --- payu/experiment.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/payu/experiment.py b/payu/experiment.py index 9628ad47..85c88300 100644 --- a/payu/experiment.py +++ b/payu/experiment.py @@ -237,9 +237,11 @@ def load_modules(self): loaded_mods = os.environ.get('LOADEDMODULES', '').split(':') for mod in loaded_mods: - mod_base = mod.split('/')[0] - if mod_base not in core_modules: - envmod.module('unload', mod) + if len(mod) > 0: + print('mod '+mod) + mod_base = mod.split('/')[0] + if mod_base not in core_modules: + envmod.module('unload', mod) # Now load model-dependent modules for mod in self.modules: From fbb6b320c5bd2d921e3fe8a6de25f80dae129e5e Mon Sep 17 00:00:00 2001 From: Aidan Heerdegen Date: Fri, 29 Nov 2019 10:40:30 +1100 Subject: [PATCH 3/5] Debugging stuff to be deleted --- payu/envmod.py | 6 ++++++ payu/subcommands/run_cmd.py | 1 + 2 files changed, 7 insertions(+) diff --git a/payu/envmod.py b/payu/envmod.py index c892b9ac..db266b0f 100644 --- a/payu/envmod.py +++ b/payu/envmod.py @@ -39,6 +39,8 @@ def setup(basepath=DEFAULT_BASEPATH): os.environ['MODULE_VERSION_STACK'] = module_version os.environ['MODULESHOME'] = moduleshome + print(os.environ) + if 'MODULEPATH' not in os.environ: module_initpath = os.path.join(moduleshome, 'init', '.modulespath') with open(module_initpath) as initpaths: @@ -82,6 +84,8 @@ def module(command, *args): cmd = '{0} python {1} {2}'.format(modulecmd, command, ' '.join(args)) + print(cmd) + envs, _ = subprocess.Popen(shlex.split(cmd), stdout=subprocess.PIPE).communicate() exec(envs) @@ -105,6 +109,8 @@ def lib_update(bin_path, lib_name): # pylint: disable=unbalanced-tuple-unpacking mod_name, mod_version = fsops.splitpath(lib_path)[2:4] + print(mod_name, mod_version, lib_path) + module('unload', mod_name) module('load', os.path.join(mod_name, mod_version)) return '{0}/{1}'.format(mod_name, mod_version) diff --git a/payu/subcommands/run_cmd.py b/payu/subcommands/run_cmd.py index ca559dac..20851bef 100644 --- a/payu/subcommands/run_cmd.py +++ b/payu/subcommands/run_cmd.py @@ -106,6 +106,7 @@ def runcmd(model_type, config_path, init_run, n_runs, lab_path, reproduce): def runscript(): + print('modulepath = {}'.format(os.environ.get('MODULEPATH',None))) parser = argparse.ArgumentParser() for arg in arguments: parser.add_argument(*arg['flags'], **arg['parameters']) From f6f2d244d0a6aac343e8022586369aca9e0d7f49 Mon Sep 17 00:00:00 2001 From: Aidan Heerdegen Date: Fri, 29 Nov 2019 11:19:37 +1100 Subject: [PATCH 4/5] Removed debugging code --- payu/envmod.py | 6 ------ payu/subcommands/run_cmd.py | 1 - 2 files changed, 7 deletions(-) diff --git a/payu/envmod.py b/payu/envmod.py index db266b0f..c892b9ac 100644 --- a/payu/envmod.py +++ b/payu/envmod.py @@ -39,8 +39,6 @@ def setup(basepath=DEFAULT_BASEPATH): os.environ['MODULE_VERSION_STACK'] = module_version os.environ['MODULESHOME'] = moduleshome - print(os.environ) - if 'MODULEPATH' not in os.environ: module_initpath = os.path.join(moduleshome, 'init', '.modulespath') with open(module_initpath) as initpaths: @@ -84,8 +82,6 @@ def module(command, *args): cmd = '{0} python {1} {2}'.format(modulecmd, command, ' '.join(args)) - print(cmd) - envs, _ = subprocess.Popen(shlex.split(cmd), stdout=subprocess.PIPE).communicate() exec(envs) @@ -109,8 +105,6 @@ def lib_update(bin_path, lib_name): # pylint: disable=unbalanced-tuple-unpacking mod_name, mod_version = fsops.splitpath(lib_path)[2:4] - print(mod_name, mod_version, lib_path) - module('unload', mod_name) module('load', os.path.join(mod_name, mod_version)) return '{0}/{1}'.format(mod_name, mod_version) diff --git a/payu/subcommands/run_cmd.py b/payu/subcommands/run_cmd.py index 20851bef..ca559dac 100644 --- a/payu/subcommands/run_cmd.py +++ b/payu/subcommands/run_cmd.py @@ -106,7 +106,6 @@ def runcmd(model_type, config_path, init_run, n_runs, lab_path, reproduce): def runscript(): - print('modulepath = {}'.format(os.environ.get('MODULEPATH',None))) parser = argparse.ArgumentParser() for arg in arguments: parser.add_argument(*arg['flags'], **arg['parameters']) From ef4104bf143e5e10c7fa56035d14efb17ad3b5b1 Mon Sep 17 00:00:00 2001 From: Aidan Heerdegen Date: Fri, 29 Nov 2019 11:20:36 +1100 Subject: [PATCH 5/5] PEP8 fixes --- payu/experiment.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/payu/experiment.py b/payu/experiment.py index 85c88300..b2bacd56 100644 --- a/payu/experiment.py +++ b/payu/experiment.py @@ -237,7 +237,7 @@ def load_modules(self): loaded_mods = os.environ.get('LOADEDMODULES', '').split(':') for mod in loaded_mods: - if len(mod) > 0: + if len(mod) > 0: print('mod '+mod) mod_base = mod.split('/')[0] if mod_base not in core_modules: