Added routine to return correct storage string and test

.travis.yml

@@ -14,7 +14,7 @@ before_script:
 script:
     - payu list
     - pylint --extension-pkg-whitelist=netCDF4 -E payu
-    - if [[ $TRAVIS_PYTHON_VERSION == 3.6 || $TRAVIS_PYTHON_VERSION == 3.7 ]]; then PYTHONPATH=$(pwd) coverage run --source payu -m py.test test/*.py; fi;
+    - if [[ $TRAVIS_PYTHON_VERSION == 3.6 || $TRAVIS_PYTHON_VERSION == 3.7 ]]; then PYTHONPATH=$(pwd) coverage run --source payu -m py.test -s test/*.py; fi;
 after_success:
     - coverage report -m
     - coveralls

payu/cli.py

@@ -20,7 +20,7 @@
 import payu.envmod as envmod
 from payu.models import index as supported_models
 import payu.subcommands
-from payu.scheduler.pbs import pbs_env_init
+from payu.scheduler.pbs import generate_command
 
 # Default configuration
 DEFAULT_CONFIG = 'config.yaml'
@@ -128,82 +128,11 @@ def set_env_vars(init_run=None, n_runs=None, lab_path=None, dir_path=None,
     return payu_env_vars
 
 
-def submit_job(pbs_script, pbs_config, pbs_vars=None):
+def submit_job(script, config, vars=None):
     """Submit a userscript the scheduler."""
 
-    pbs_env_init()
+    cmd = generate_command(script, config, vars)
 
-    # Initialisation
-    if pbs_vars is None:
-        pbs_vars = {}
-
-    pbs_flags = []
-
-    pbs_queue = pbs_config.get('queue', 'normal')
-    pbs_flags.append('-q {queue}'.format(queue=pbs_queue))
-
-    pbs_project = pbs_config.get('project', os.environ['PROJECT'])
-    pbs_flags.append('-P {project}'.format(project=pbs_project))
-
-    pbs_resources = ['walltime', 'ncpus', 'mem', 'jobfs']
-
-    for res_key in pbs_resources:
-        res_flags = []
-        res_val = pbs_config.get(res_key)
-        if res_val:
-            res_flags.append('{key}={val}'.format(key=res_key, val=res_val))
-
-        if res_flags:
-            pbs_flags.append('-l {res}'.format(res=','.join(res_flags)))
-
-    # TODO: Need to pass lab.config_path somehow...
-    pbs_jobname = pbs_config.get('jobname', os.path.basename(os.getcwd()))
-    if pbs_jobname:
-        # PBSPro has a 15-character jobname limit
-        pbs_flags.append('-N {name}'.format(name=pbs_jobname[:15]))
-
-    pbs_priority = pbs_config.get('priority')
-    if pbs_priority:
-        pbs_flags.append('-p {priority}'.format(priority=pbs_priority))
-
-    pbs_flags.append('-l wd')
-
-    pbs_join = pbs_config.get('join', 'n')
-    if pbs_join not in ('oe', 'eo', 'n'):
-        print('payu: error: unknown qsub IO stream join setting.')
-        sys.exit(-1)
-    else:
-        pbs_flags.append('-j {join}'.format(join=pbs_join))
-
-    # Append environment variables to qsub command
-    # TODO: Support full export of environment variables: `qsub -V`
-    pbs_vstring = ','.join('{0}={1}'.format(k, v)
-                           for k, v in pbs_vars.items())
-    pbs_flags.append('-v ' + pbs_vstring)
-
-    # Append any additional qsub flags here
-    pbs_flags_extend = pbs_config.get('qsub_flags')
-    if pbs_flags_extend:
-        pbs_flags.append(pbs_flags_extend)
-
-    if not os.path.isabs(pbs_script):
-        # NOTE: PAYU_PATH is always set if `set_env_vars` was always called.
-        #       This is currently always true, but is not explicitly enforced.
-        #       So this conditional check is a bit redundant.
-        payu_bin = pbs_vars.get('PAYU_PATH', os.path.dirname(sys.argv[0]))
-        pbs_script = os.path.join(payu_bin, pbs_script)
-        assert os.path.isfile(pbs_script)
-
-    # Set up environment modules here for PBS.
-    envmod.setup()
-    envmod.module('load', 'pbs')
-
-    # Construct job submission command
-    cmd = 'qsub {flags} -- {python} {script}'.format(
-        flags=' '.join(pbs_flags),
-        python=sys.executable,
-        script=pbs_script
-    )
     print(cmd)
 
     subprocess.check_call(shlex.split(cmd))

payu/envmod.py

@@ -32,8 +32,13 @@ def setup(basepath=DEFAULT_BASEPATH):
     if not os.path.isdir(moduleshome):
         print('payu: warning: MODULESHOME does not exist; disabling '
               'environment modules.')
-        os.environ['MODULESHOME'] = ''
+        try:
+            del(os.environ['MODULESHOME'])
+        except KeyError:
+            pass
         return
+    else:
+        print('payu: Found modules in {}'.format(moduleshome))
 
     os.environ['MODULE_VERSION'] = module_version
     os.environ['MODULE_VERSION_STACK'] = module_version

payu/experiment.py

@@ -92,7 +92,8 @@ def __init__(self, lab, reproduce=None):
             reproduce = os.environ.get('PAYU_REPRODUCE', False)
 
         # Initialize manifest
-        self.manifest = Manifest(self, reproduce=reproduce)
+        self.manifest = Manifest(self.config.get('manifest', {}),
+                                 reproduce=reproduce)
 
         # Miscellaneous configurations
         # TODO: Move this stuff somewhere else

payu/fsops.py

@@ -124,3 +124,13 @@ def patch_lustre_path(f_path):
             f_path = './' + f_path
 
     return f_path
+
+
+def check_exe_path(payu_path, pbs_script):
+    """Check a payu executable path is locateable """
+    if not os.path.isabs(pbs_script):
+        pbs_script = os.path.join(payu_path, pbs_script)
+
+    assert os.path.isfile(pbs_script)
+
+    return pbs_script

payu/manifest.py

@@ -186,6 +186,7 @@ def make_link(self, filepath):
                 print('removing from manifest')
                 self.delete(filepath)
                 self.needsync = True
+                self.existing_filepaths.discard(filepath)
         else:
             try:
                 destdir = os.path.dirname(filepath)
@@ -208,6 +209,8 @@ def make_link(self, filepath):
                                               orig=self.fullpath(filepath),
                                               local=filepath))
                 raise
+            finally:
+                self.existing_filepaths.discard(filepath)
 
     def make_links(self):
         """
@@ -222,20 +225,23 @@ def copy(self, path):
         """
         shutil.copy(self.path, path)
 
+    def get_fullpaths(self):
+        files = []
+        for filepath in list(self):
+            files.append(self.fullpath(filepath))
+        return files
+
 
 class Manifest(object):
     """
     A Manifest class which stores all manifests for file tracking and
     methods to operate on them
     """
 
-    def __init__(self, expt, reproduce):
-
-        # Inherit experiment configuration
-        self.expt = expt
+    def __init__(self, config, reproduce):
 
         # Manifest control configuration
-        self.manifest_config = self.expt.config.get('manifest', {})
+        self.manifest_config = config
 
         # Not currently supporting specifying hash functions
         # self.hash_functions = manifest_config.get(
@@ -247,29 +253,33 @@ def __init__(self, expt, reproduce):
         if isinstance(self.ignore, str):
             self.ignore = [self.ignore]
 
-        # Initialise manifests
+        # Initialise manifests and reproduce flags
         self.manifests = {}
-        for mf in ['input', 'restart', 'exe']:
-            self.manifests[mf] = PayuManifest(
-                os.path.join('manifests', '{}.yaml'.format(mf)),
-                ignore=self.ignore
-            )
-
         self.have_manifest = {}
-        for mf in self.manifests:
-            self.have_manifest[mf] = False
-
-        # Set reproduce flags
-        self.reproduce_config = self.manifest_config.get('reproduce', {})
+        reproduce_config = self.manifest_config.get('reproduce', {})
         self.reproduce = {}
-        for mf in self.manifests.keys():
-            self.reproduce[mf] = self.reproduce_config.get(mf, reproduce)
+        for mf in ['input', 'restart', 'exe']:
+            self.init_mf(mf)
+            self.reproduce[mf] = reproduce_config.get(mf, reproduce)
 
         # Make sure the manifests directory exists
         mkdir_p(os.path.dirname(self.manifests['exe'].path))
 
+        # Set flag to auto-scan input directories
         self.scaninputs = self.manifest_config.get('scaninputs', True)
 
+        if self.reproduce['input'] and self.scaninputs:
+            print("scaninputs set to False when reproduce input is True")
+            self.scaninputs = False
+
+    def init_mf(self, mf):
+        # Initialise a sub-manifest object
+        self.manifests[mf] = PayuManifest(
+            os.path.join('manifests', '{}.yaml'.format(mf)),
+            ignore=self.ignore
+        )
+        self.have_manifest[mf] = False
+
     def __iter__(self):
         """
         Iterator method
@@ -281,78 +291,62 @@ def __len__(self):
         """Return the number of manifests in the manifest class."""
         return len(self.manifests)
 
-    def setup(self):
-
-        if (os.path.exists(self.manifests['input'].path)):
-            # Always read input manifest if available
-            try:
-                print('Loading input manifest: {path}'
-                      ''.format(path=self.manifests['input'].path))
-                self.manifests['input'].load()
-
-                if len(self.manifests['input']) > 0:
-                    self.have_manifest['input'] = True
-                    if self.scaninputs:
-                        # Save existing filepath information
-                        self.manifests['input'].existing_filepaths = \
-                            set(self.manifests['input'].data.keys())
-                    else:
-                        # Input directories not scanned. Populate
-                        # inputs in workdir using input manifest
-                        print('Making input links from manifest'
-                              '(scaninputs=False)')
-                        self.manifests['input'].make_links()
-            except Exception as e:
-                print("Error loading input manifest: {}".format(e))
-                self.manifests['input'].have_manifest = False
-            finally:
-                self.manifests['input'].have_manifest = True
-
-        if self.reproduce['exe']:
-            # Only load existing exe manifest if reproduce. Trivial to
-            # recreate and means no check required for changed
-            # executable paths
-            if os.path.exists(self.manifests['exe'].path):
-                # Read manifest
-                print('Loading exe manifest: {}'
-                      .format(self.manifests['exe'].path))
-                self.manifests['exe'].load()
-
-                if len(self.manifests['exe']) > 0:
-                    self.have_manifest['exe'] = True
-
-            # Must make links as no files will be added to the manifest
-            print('Making exe links')
-            self.manifests['exe'].make_links()
-        else:
-            self.have_manifest['exe'] = False
-
-        if self.reproduce['restart']:
-            # Only load restart manifest if reproduce. Normally want to
-            # scan for new restarts
-
-            # Read restart manifest
-            print('Loading restart manifest: {}'
-                  ''.format(self.have_manifest['restart']))
-            self.manifests['restart'].load()
-
-            if len(self.manifests['restart']) > 0:
-                self.have_manifest['restart'] = True
-
-            for model in self.expt.models:
-                model.have_restart_manifest = True
+    def load(self):
+        """
+        Load manifests
+        """
+        for mf in self.manifests:
+            self.have_manifest[mf] = False
+            if (os.path.exists(self.manifests[mf].path)):
+                try:
+                    print('Loading {mf} manifest: {path}'
+                          ''.format(mf=mf, path=self.manifests[mf].path))
+                    self.manifests[mf].load()
+                except Exception as e:
+                    print('Error loading {mf} manifest: '
+                          '{error}'.format(mf=mf, error=e))
+                finally:
+                    if len(self.manifests[mf]) > 0:
+                        self.have_manifest[mf] = True
 
-            # Must make links as no files will be added to the manifest
-            print('Making restart links')
-            self.manifests['restart'].make_links()
-        else:
-            self.have_manifest['restart'] = False
+    def setup(self):
 
+        # Load all available manifests
+        self.load()
+
+        if self.have_manifest['input']:
+            if self.scaninputs:  # Must be False for reproduce=True
+                # Save existing filepath information
+                self.manifests['input'].existing_filepaths = \
+                    set(self.manifests['input'].data.keys())
+
+        if self.have_manifest['exe']:
+            if not self.reproduce['exe']:
+                # Re-initialise exe manifest. Trivial to recreate
+                # and means no check required for changed executable
+                # paths
+                self.init_mf('exe')
+
+        if self.have_manifest['restart']:
+            if not self.reproduce['restart']:
+                # Re-initialise restart manifest. Only keep restart manifest
+                # if reproduce. Normally want to scan for new restarts
+                self.init_mf('restart')
+
+        # Check to make all manifests that should be populated are and
+        # make links in work directory for existing manifests
         for mf in self.manifests.keys():
-            if self.reproduce[mf] and not self.have_manifest[mf]:
-                print('{} manifest must exist if reproduce is True'
-                      ''.format(mf.capitalize()))
-                exit(1)
+            if self.have_manifest[mf]:
+                # Don't make links for inputs when scaninputs is True
+                if mf == 'input' and self.scaninputs:
+                    break
+                print('Making {} links'.format(mf))
+                self.manifests[mf].make_links()
+            else:
+                if self.reproduce[mf]:
+                    print('{} manifest must exist if reproduce is True'
+                          ''.format(mf.capitalize()))
+                    exit(1)
 
     def check_manifests(self):
 
@@ -402,3 +396,12 @@ def add_filepath(self, manifest, filepath, fullpath, copy=False):
         if self.manifests[manifest].add_filepath(filepath, fullpath, copy):
             # Only link if filepath was added
             self.manifests[manifest].make_link(filepath)
+
+    def get_all_fullpaths(self):
+        """
+        Return a list of all fullpaths in manifest files
+        """
+        files = []
+        for mf in self.manifests:
+            files.extend(self.manifests[mf].get_fullpaths())
+        return files

payu/models/model.py

@@ -269,7 +269,8 @@ def setup(self):
                             workrelpath,
                             f_name
                         )
-                        # Do not use input file if it is in RESTART
+                        # Do not use input file if already linked
+                        # as a restart file
                         if not os.path.exists(f_link):
                             self.expt.manifest.add_filepath(
                                 'input',