Merge 45e1abb into 0ca7b9e

.circleci/config.yml

@@ -13,9 +13,9 @@ jobs:
             - run: |
                 wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O conda.sh
                 bash conda.sh -b -p ~/conda
+                ~/conda/bin/conda update --all
                 ~/conda/bin/conda config --system --add channels conda-forge
                 ~/conda/bin/conda config --system --add channels coecms
-                ~/conda/bin/conda update conda
                 ~/conda/bin/conda install --yes conda-build conda-verify
 
             - run: |

.travis.yml

@@ -1,18 +1,20 @@
 language: python
 python:
     - "2.7"
-    - "3.5"
     - "3.6"
+    - "3.7"
 # Disabling Torque setup until we can get this working on Travis
-before_install:
-    - sudo apt-get install torque-server torque-client torque-mom torque-pam
-    - sudo ./test/setup_torque.sh
+# before_install:
+#     - sudo apt-get install torque-server torque-client torque-mom torque-pam
+#     - sudo ./test/setup_torque.sh
 install:
     - pip install .
 before_script:
     - pip install -r test/requirements_test.txt
 script:
-    - PYTHONPATH=$(pwd) coverage run --source payu bin/payu
+    - payu list
+    - pylint --extension-pkg-whitelist=netCDF4 -E payu
+    - if [[ $TRAVIS_PYTHON_VERSION == 3.6 || $TRAVIS_PYTHON_VERSION == 3.7 ]]; then PYTHONPATH=$(pwd) coverage run --source payu -m py.test test/test_payu.py test/test_manifest.py; fi;
 after_success:
     - coverage report -m
     - coveralls

README.rst

@@ -1,8 +1,13 @@
 Payu
 ====
 
-.. image:: https://travis-ci.org/marshallward/payu.svg?branch=master
-   :target: https://travis-ci.org/marshallward/payu
+.. image:: https://travis-ci.org/payu-org/payu.svg?branch=master
+    :target: https://travis-ci.org/payu-org/payu
+.. image:: https://coveralls.io/repos/github/payu-org/payu/badge.svg?branch=master
+    :target: https://coveralls.io/github/payu-org/payu?branch=master
+
+
+
 
 Payu is a climate model workflow management tool for supercomputing
 environments.

conda/meta.yaml

@@ -14,6 +14,7 @@ requirements:
     build:
         - python
         - pbr
+        - setuptools
     run:
         - python
         - f90nml >=0.16

payu/cli.py

@@ -20,6 +20,7 @@
 import payu.envmod as envmod
 from payu.models import index as supported_models
 import payu.subcommands
+from payu.scheduler.pbs import pbs_env_init
 
 # Default configuration
 DEFAULT_CONFIG = 'config.yaml'
@@ -124,6 +125,8 @@ def set_env_vars(init_run=None, n_runs=None, lab_path=None, dir_path=None,
 def submit_job(pbs_script, pbs_config, pbs_vars=None):
     """Submit a userscript the scheduler."""
 
+    pbs_env_init()
+
     # Initialisation
     if pbs_vars is None:
         pbs_vars = {}

payu/envmod.py

@@ -98,6 +98,7 @@ def lib_update(bin_path, lib_name):
         if lib_name in lib_entry:
             lib_path = lib_entry.split()[2]
 
+            # pylint: disable=unbalanced-tuple-unpacking
             mod_name, mod_version = fsops.splitpath(lib_path)[2:4]
 
             module('unload', mod_name)

payu/experiment.py

@@ -116,8 +116,6 @@ def __init__(self, lab, reproduce=None):
 
         self.run_id = None
 
-        pbs_env_init()
-
     def init_models(self):
 
         self.model_name = self.config.get('model')
@@ -657,7 +655,7 @@ def run(self, *user_flags):
             job_id = get_job_id(short=False)
 
             if job_id == '':
-                job_id = self.run_id[:6]
+                job_id = str(self.run_id)[:6]
 
             for fname in self.output_fnames:
 
@@ -891,17 +889,16 @@ def run_userscript(self, script_cmd):
         # First try to interpret the argument as a full command:
         try:
             sp.check_call(shlex.split(script_cmd))
-        except (EnvironmentError, sp.CalledProcessError) as exc:
+        except EnvironmentError as exc:
             # Now try to run the script explicitly
-            if isinstance(exc, EnvironmentError) and exc.errno == errno.ENOENT:
+            if exc.errno == errno.ENOENT:
                 cmd = os.path.join(self.control_path, script_cmd)
                 # Simplistic recursion check
                 assert os.path.isfile(cmd)
                 self.run_userscript(cmd)
 
             # If we get a "non-executable" error, then guess the type
-            elif (isinstance(exc, EnvironmentError)
-                    and exc.errno == errno.EACCES):
+            elif exc.errno == errno.EACCES:
                 # TODO: Move outside
                 ext_cmd = {'.py': sys.executable,
                            '.sh': '/bin/bash',
@@ -919,15 +916,10 @@ def run_userscript(self, script_cmd):
                 else:
                     # If we can't guess the shell, then abort
                     raise
-
+        except sp.CalledProcessError as exc:
             # If the script runs but the output is bad, then warn the user
-            elif type(exc) == sp.CalledProcessError:
-                print('payu: warning: user script \'{0}\' failed (error {1}).'
-                      ''.format(script_cmd, exc.returncode))
-
-            # If all else fails, raise an error
-            else:
-                raise
+            print('payu: warning: user script \'{0}\' failed (error {1}).'
+                  ''.format(script_cmd, exc.returncode))
 
     def sweep(self, hard_sweep=False):
         # TODO: Fix the IO race conditions!

payu/laboratory.py

@@ -48,6 +48,12 @@ def __init__(self, model_type=None, config_path=None, lab_path=None):
         self.input_basepath = os.path.join(self.basepath, 'input')
         self.work_path = os.path.join(self.basepath, 'work')
 
+        print("laboratory path: ", self.basepath)
+        print("binary path: ", self.bin_path)
+        print("input path: ", self.input_basepath)
+        print("work path: ", self.work_path)
+        print("archive path: ", self.archive_path)
+
     def get_default_lab_path(self, config):
         """Generate a default laboratory path based on user environment."""
         # Default path settings

payu/manifest.py

@@ -15,11 +15,10 @@
 import sys
 import shutil
 import stat
-from distutils.dir_util import mkpath
 
 from yamanifest.manifest import Manifest as YaManifest
 
-from payu.fsops import make_symlink
+from payu.fsops import make_symlink, mkdir_p
 
 
 # fast_hashes = ['nchash','binhash']
@@ -40,6 +39,7 @@ def __init__(self, path, hashes=None, ignore=None, **kwargs):
             self.ignore = ignore
 
         self.needsync = False
+        self.existing_filepaths = set()
 
     def check_fast(self, reproduce=False, **args):
         """
@@ -147,9 +147,8 @@ def add_filepath(self, filepath, fullpath, copy=False):
         if copy:
             self.data[filepath]['copy'] = copy
 
-        if hasattr(self, 'existing_filepaths'):
-            if filepath in self.existing_filepaths:
-                self.existing_filepaths.remove(filepath)
+        if filepath in self.existing_filepaths:
+            self.existing_filepaths.remove(filepath)
 
         return True
 
@@ -234,20 +233,10 @@ def __init__(self, expt, reproduce):
 
         # Inherit experiment configuration
         self.expt = expt
-        self.reproduce = reproduce
 
         # Manifest control configuration
         self.manifest_config = self.expt.config.get('manifest', {})
 
-        # If the run sets reproduce, default to reproduce executables. Allow
-        # user to specify not to reproduce executables (might not be feasible
-        # if executables don't match platform, or desirable if bugs existed in
-        # old exe)
-        self.reproduce_exe = (
-            self.reproduce and
-            self.manifest_config.get('reproduce_exe', True)
-        )
-
         # Not currently supporting specifying hash functions
         # self.hash_functions = manifest_config.get(
         #     'hashfns',
@@ -270,8 +259,14 @@ def __init__(self, expt, reproduce):
         for mf in self.manifests:
             self.have_manifest[mf] = False
 
+        # Set reproduce flags
+        self.reproduce_config = self.manifest_config.get('reproduce', {})
+        self.reproduce = {}
+        for mf in self.manifests.keys():
+            self.reproduce[mf] = self.reproduce_config.get(mf, reproduce)
+
         # Make sure the manifests directory exists
-        mkpath(os.path.dirname(self.manifests['exe'].path))
+        mkdir_p(os.path.dirname(self.manifests['exe'].path))
 
         self.scaninputs = self.manifest_config.get('scaninputs', True)
 
@@ -287,28 +282,36 @@ def __len__(self):
         return len(self.manifests)
 
     def setup(self):
-        # Check if manifest files exist
-        self.have_manifest['restart'] = os.path.exists(
-            self.manifests['restart'].path
-        )
-
-        if (os.path.exists(self.manifests['input'].path) and
-                not self.manifest_config.get('overwrite', False)):
-            # Read manifest
-            print('Loading input manifest: {path}'
-                  ''.format(path=self.manifests['input'].path))
-            self.manifests['input'].load()
-
-            if len(self.manifests['input']) > 0:
-                self.have_manifest['input'] = True
-                if self.scaninputs:
-                    self.manifests['input'].existing_filepaths = \
-                        set(self.manifests['input'].data.keys())
-
-        if self.reproduce:
 
+        if (os.path.exists(self.manifests['input'].path)):
+            # Always read input manifest if available
+            try:
+                print('Loading input manifest: {path}'
+                      ''.format(path=self.manifests['input'].path))
+                self.manifests['input'].load()
+
+                if len(self.manifests['input']) > 0:
+                    self.have_manifest['input'] = True
+                    if self.scaninputs:
+                        # Save existing filepath information
+                        self.manifests['input'].existing_filepaths = \
+                            set(self.manifests['input'].data.keys())
+                    else:
+                        # Input directories not scanned. Populate
+                        # inputs in workdir using input manifest
+                        print('Making input links from manifest'
+                              '(scaninputs=False)')
+                        self.manifests['input'].make_links()
+            except Exception as e:
+                print("Error loading input manifest: {}".format(e))
+                self.manifests['input'].have_manifest = False
+            finally:
+                self.manifests['input'].have_manifest = True
+
+        if self.reproduce['exe']:
             # Only load existing exe manifest if reproduce. Trivial to
-            # recreate and no check required for changed executable paths
+            # recreate and means no check required for changed
+            # executable paths
             if os.path.exists(self.manifests['exe'].path):
                 # Read manifest
                 print('Loading exe manifest: {}'
@@ -318,6 +321,16 @@ def setup(self):
                 if len(self.manifests['exe']) > 0:
                     self.have_manifest['exe'] = True
 
+            # Must make links as no files will be added to the manifest
+            print('Making exe links')
+            self.manifests['exe'].make_links()
+        else:
+            self.have_manifest['exe'] = False
+
+        if self.reproduce['restart']:
+            # Only load restart manifest if reproduce. Normally want to
+            # scan for new restarts
+
             # Read restart manifest
             print('Loading restart manifest: {}'
                   ''.format(self.have_manifest['restart']))
@@ -326,56 +339,44 @@ def setup(self):
             if len(self.manifests['restart']) > 0:
                 self.have_manifest['restart'] = True
 
-            # MUST have input and restart manifests to be able to reproduce run
-            for mf in ['restart', 'input']:
-                if not self.have_manifest[mf]:
-                    print('{} manifest cannot be empty if reproduce is True'
-                          ''.format(mf.capitalize()))
-                    exit(1)
-
-            if self.reproduce_exe and not self.have_manifest['exe']:
-                print('Executable manifest cannot empty if reproduce and '
-                      'reproduce_exe are True')
-                exit(1)
-
-            # Must make links as no files will be added to the manifest
-            for mf in ['exe', 'restart', 'input']:
-                print('Making links: {}'.format(mf))
-                self.manifests[mf].make_links()
-
             for model in self.expt.models:
                 model.have_restart_manifest = True
 
+            # Must make links as no files will be added to the manifest
+            print('Making restart links')
+            self.manifests['restart'].make_links()
         else:
             self.have_manifest['restart'] = False
 
-            if not self.scaninputs:
-                # If input directories not scanned then the only
-                # way to populate the inputs in work is to rely
-                # on input manifest
-                print('Making links from input manifest (scaninputs=False)')
-                self.manifests['input'].make_links()
+        for mf in self.manifests.keys():
+            if self.reproduce[mf] and not self.have_manifest[mf]:
+                print('{} manifest must exist if reproduce is True'
+                      ''.format(mf.capitalize()))
+                exit(1)
 
     def check_manifests(self):
 
         print("Checking exe and input manifests")
-        self.manifests['exe'].check_fast(reproduce=self.reproduce_exe)
-        if hasattr(self.manifests['input'], 'existing_filepaths'):
-            # Delete filepaths from input manifest
-            for filepath in self.manifests['input'].existing_filepaths:
-                print('File no longer in input directory: {file} '
-                      'removing from manifest'.format(file=filepath))
-                self.manifests['input'].delete(filepath)
-            self.manifests['input'].needsync = True
-
-        self.manifests['input'].check_fast(reproduce=self.reproduce)
-
-        if self.reproduce:
+        self.manifests['exe'].check_fast(reproduce=self.reproduce['exe'])
+
+        if not self.reproduce['input']:
+            if len(self.manifests['input'].existing_filepaths) > 0:
+                # Delete missing filepaths from input manifest
+                for filepath in self.manifests['input'].existing_filepaths:
+                    print('File no longer in input directory: {file} '
+                          'removing from manifest'.format(file=filepath))
+                    self.manifests['input'].delete(filepath)
+                self.manifests['input'].needsync = True
+
+        self.manifests['input'].check_fast(reproduce=self.reproduce['input'])
+
+        if self.reproduce['restart']:
             print("Checking restart manifest")
         else:
             print("Creating restart manifest")
             self.manifests['restart'].needsync = True
-        self.manifests['restart'].check_fast(reproduce=self.reproduce)
+        self.manifests['restart'].check_fast(
+                reproduce=self.reproduce['restart'])
 
         # Write updates to version on disk
         for mf in self.manifests:
@@ -385,7 +386,7 @@ def check_manifests(self):
 
     def copy_manifests(self, path):
 
-        mkpath(path)
+        mkdir_p(path)
         try:
             for mf in self.manifests:
                 self.manifests[mf].copy(path)