Merge 1e4f758 into 98bd6ca

.travis.yml

@@ -14,7 +14,7 @@ before_script:
 script:
     - payu list
     - pylint --extension-pkg-whitelist=netCDF4 -E payu
-    - if [[ $TRAVIS_PYTHON_VERSION == 3.6 || $TRAVIS_PYTHON_VERSION == 3.7 ]]; then PYTHONPATH=$(pwd) coverage run --source payu -m py.test test/test_payu.py test/test_manifest.py; fi;
+    - if [[ $TRAVIS_PYTHON_VERSION == 3.6 || $TRAVIS_PYTHON_VERSION == 3.7 ]]; then PYTHONPATH=$(pwd) coverage run --source payu -m py.test test/*.py; fi;
 after_success:
     - coverage report -m
     - coveralls

payu/laboratory.py

@@ -37,6 +37,14 @@ def __init__(self, model_type=None, config_path=None, lab_path=None):
         else:
             self.basepath = lab_path
 
+        # Support multiple bases for default short/scratch
+        # locations. Fall back to control directory if others
+        # don't exist
+        for path in ['/short', '/scratch', '.']:
+            if os.path.exists(path):
+                self.base = path
+                break
+
         # If no lab path is set, generate a default path
         if not self.basepath:
             self.basepath = self.get_default_lab_path(config)
@@ -60,7 +68,7 @@ def get_default_lab_path(self, config):
 
         # Append project name if present (NCI-specific)
         default_project = os.environ.get('PROJECT', '')
-        default_short_path = os.path.join('/short', default_project)
+        default_short_path = os.path.join(self.base, default_project)
         default_user = pwd.getpwuid(os.getuid()).pw_name
 
         short_path = config.get('shortpath', default_short_path)

test/common.py

@@ -4,6 +4,39 @@
 
 import yaml
 
+import payu
+
+# Namespace clash if import setup_cmd.runcmd as setup. For
+# consistency use payu_ prefix for all commands
+from payu.subcommands.init_cmd import runcmd as payu_init
+from payu.subcommands.setup_cmd import runcmd as payu_setup_orignal
+from payu.subcommands.sweep_cmd import runcmd as payu_sweep
+
+tmpdir = Path().cwd() / Path('test') / 'tmp'
+ctrldir = tmpdir / 'ctrl'
+labdir = tmpdir / 'lab'
+workdir = ctrldir / 'work'
+
+config = {
+            'shortpath': '..',
+            'laboratory': 'lab',
+            'queue': 'normal',
+            'project': 'aa30',
+            'walltime': '0:30:00',
+            'ncpus': 64,
+            'mem': '64GB',
+            'jobname': 'testrun',
+            'model': 'test',
+            'exe': 'test.exe',
+            'input': 'testrun_1',
+            'manifest': {
+                        'scaninputs': False,
+                        'reproduce': {
+                                        'input': False,
+                                        'exe': False
+                                        }
+                        }
+            }
 
 @contextmanager
 def cd(directory):
@@ -38,3 +71,71 @@ def get_manifests(mfdir):
         with mfpath.open() as fh:
             manifests[mfpath.name] = list(yaml.safe_load_all(fh))[1]
     return manifests
+
+
+def sweep_work(hard_sweep=False):
+    # Sweep workdir
+    with cd(ctrldir):
+        payu_sweep(model_type=None,
+                   config_path=None,
+                   hard_sweep=hard_sweep,
+                   lab_path=str(labdir))
+
+
+def payu_setup(model_type=None,
+               config_path=None,
+               lab_path=None,
+               force_archive=None,
+               reproduce=None):
+    """
+    Wrapper around original setup command to provide default arguments
+    and run in ctrldir
+    """
+    with cd(ctrldir):
+        payu_sweep(model_type=None,
+                   config_path=None,
+                   hard_sweep=False,
+                   lab_path=str(labdir))
+        payu_setup_orignal(model_type,
+                           config_path,
+                           lab_path,
+                           force_archive,
+                           reproduce)
+
+
+def write_config():
+    with (ctrldir / 'config.yaml').open('w') as file:
+        file.write(yaml.dump(config, default_flow_style=False))
+
+
+def make_exe():
+    # Create a fake executable file
+    bindir = labdir / 'bin'
+    bindir.mkdir(parents=True, exist_ok=True)
+    exe = config['exe']
+    exe_size = 199
+    make_random_file(bindir/exe, exe_size)
+
+
+def make_inputs():
+    # Create some fake input files
+    inputdir = labdir / 'input' / config['input']
+    inputdir.mkdir(parents=True, exist_ok=True)
+    for i in range(1, 4):
+        make_random_file(inputdir/'input_00{i}.bin'.format(i=i),
+                         1000**2 + i)
+
+
+def make_restarts():
+    # Create some fake restart files
+    restartdir = labdir / 'archive' / 'restarts'
+    restartdir.mkdir(parents=True, exist_ok=True)
+    for i in range(1, 4):
+        make_random_file(restartdir/'restart_00{i}.bin'.format(i=i),
+                         5000**2 + i)
+
+
+def make_all_files():
+    make_inputs()
+    make_exe()
+    make_restarts()

test/test_manifest.py

@@ -9,118 +9,16 @@
 
 import pdb
 
-# Namespace clash if import setup_cmd.runcmd as setup. For
-# consistency use payu_ prefix for all commands
-from payu.subcommands.init_cmd import runcmd as payu_init
-from payu.subcommands.setup_cmd import runcmd as payu_setup_orignal
-from payu.subcommands.sweep_cmd import runcmd as payu_sweep
-
 import payu.models.test
 
 from common import cd, make_random_file, get_manifests
-
+from common import tmpdir, ctrldir, labdir, workdir
+from common import config, sweep_work, payu_init, payu_setup
+from common import write_config
+from common import make_exe, make_inputs, make_restarts, make_all_files
 
 verbose = True
 
-tmpdir = Path().cwd() / Path('test') / 'tmp'
-ctrldir = tmpdir / 'ctrl'
-labdir = tmpdir / 'lab'
-workdir = ctrldir / 'work'
-
-print(tmpdir)
-print(ctrldir)
-print(labdir)
-
-config = {
-            'shortpath': '..',
-            'laboratory': 'lab',
-            'queue': 'normal',
-            'project': 'aa30',
-            'walltime': '0:30:00',
-            'ncpus': 64,
-            'mem': '64GB',
-            'jobname': 'testrun',
-            'model': 'test',
-            'exe': 'test.exe',
-            'input': 'testrun_1',
-            'manifest': {
-                        'scaninputs': False,
-                        'reproduce': {
-                                        'input': False,
-                                        'exe': False
-                                        }
-                        }
-            }
-
-
-def sweep_work(hard_sweep=False):
-    # Sweep workdir
-    with cd(ctrldir):
-        payu_sweep(model_type=None,
-                   config_path=None,
-                   hard_sweep=hard_sweep,
-                   lab_path=str(labdir))
-
-
-def payu_setup(model_type=None,
-               config_path=None,
-               lab_path=None,
-               force_archive=None,
-               reproduce=None):
-    """
-    Wrapper around original setup command to provide default arguments
-    and run in ctrldir
-    """
-    with cd(ctrldir):
-        payu_sweep(model_type=None,
-                   config_path=None,
-                   hard_sweep=False,
-                   lab_path=str(labdir))
-        payu_setup_orignal(model_type,
-                           config_path,
-                           lab_path,
-                           force_archive,
-                           reproduce)
-
-
-def write_config():
-    with (ctrldir / 'config.yaml').open('w') as file:
-        file.write(yaml.dump(config, default_flow_style=False))
-
-
-def make_exe():
-    # Create a fake executable file
-    bindir = labdir / 'bin'
-    bindir.mkdir(parents=True, exist_ok=True)
-    exe = config['exe']
-    exe_size = 199
-    make_random_file(bindir/exe, exe_size)
-
-
-def make_inputs():
-    # Create some fake input files
-    inputdir = labdir / 'input' / config['input']
-    inputdir.mkdir(parents=True, exist_ok=True)
-    for i in range(1, 4):
-        make_random_file(inputdir/'input_00{i}.bin'.format(i=i),
-                         1000**2 + i)
-
-
-def make_restarts():
-    # Create some fake restart files
-    restartdir = labdir / 'archive' / 'restarts'
-    restartdir.mkdir(parents=True, exist_ok=True)
-    for i in range(1, 4):
-        make_random_file(restartdir/'restart_00{i}.bin'.format(i=i),
-                         5000**2 + i)
-
-
-def make_all_files():
-    make_inputs()
-    make_exe()
-    make_restarts()
-
-
 def setup_module(module):
     """
     Put any test-wide setup code in here, e.g. creating test files

test/test_paths.py

@@ -0,0 +1,107 @@
+import os
+from pathlib import Path
+import shutil
+
+import pdb
+import pytest
+import yaml
+
+import payu
+
+from payu.laboratory import Laboratory
+
+from common import cd, make_random_file, get_manifests
+from common import tmpdir, ctrldir, labdir, workdir
+from common import config, sweep_work, payu_init, payu_setup
+from common import write_config
+from common import make_exe, make_inputs, make_restarts, make_all_files
+
+verbose = True
+
+
+def setup_module(module):
+    """
+    Put any test-wide setup code in here, e.g. creating test files
+    """
+    if verbose:
+        print("setup_module      module:%s" % module.__name__)
+
+    # Should be taken care of by teardown, in case remnants lying around
+    try:
+        shutil.rmtree(tmpdir)
+    except FileNotFoundError:
+        pass
+
+    try:
+        tmpdir.mkdir()
+        labdir.mkdir()
+        ctrldir.mkdir()
+    except Exception as e:
+        print(e)
+
+    write_config()
+
+
+def teardown_module(module):
+    """
+    Put any test-wide teardown code in here, e.g. removing test outputs
+    """
+    if verbose:
+        print("teardown_module   module:%s" % module.__name__)
+
+    try:
+        shutil.rmtree(tmpdir)
+        print('removing tmp')
+    except Exception as e:
+        print(e)
+
+
+def test_laboratory_basepath():
+
+    # Test instantiating a Laboratory object
+    with cd(ctrldir):
+        lab = Laboratory(None, None, None)
+
+        assert(Path(lab.basepath).parts[0] == '..')
+        assert(Path(lab.basepath).parts[2] == 'lab')
+
+    # Set a PROJECT env variable to get reproducible paths
+    os.environ['PROJECT'] = 'x00'
+
+    # Repeat, but remove shortpath definition
+    # in config, so will fall through to default
+    # depending on platform
+    del(config['shortpath'])
+    write_config()
+    with cd(ctrldir):
+        lab = Laboratory(None, None, None)
+
+        shortpath = '.'
+        for path in ['/short', '/scratch']:
+            if Path(path).exists():
+                shortpath = path
+                break
+
+        assert(list(Path(lab.basepath).parents)[2] == Path(shortpath))
+        assert(Path(lab.basepath).parts[-3] == os.environ['PROJECT'])
+        assert(Path(lab.basepath).parts[-1] == 'lab')
+
+def test_laboratory_path():
+
+    # Set a PROJECT env variable to get reproducible paths
+    os.environ['PROJECT'] = 'x00'
+
+    # Set a relative laboratory name
+    labname = 'testlab'
+    config['laboratory'] = labname
+    write_config()
+    with cd(ctrldir):
+        lab = Laboratory(None, None, None)
+
+        shortpath = '.'
+        for path in ['/short', '/scratch']:
+            if Path(path).exists():
+                shortpath = path
+                break
+
+        assert(Path(lab.basepath).parts[-1] == labname)