Skip to content

Commit

Permalink
Merge 0b78a66 into 98bd6ca
Browse files Browse the repository at this point in the history
  • Loading branch information
@aidanheerdegen
aidanheerdegen committed Dec 6, 2019
2 parents 98bd6ca + 0b78a66 commit cb5f653
Show file tree
Hide file tree
Showing 3 changed files with 114 additions and 107 deletions.
10 changes: 9 additions & 1 deletion payu/laboratory.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,14 @@ def __init__(self, model_type=None, config_path=None, lab_path=None):
else:
self.basepath = lab_path

# Support multiple bases for default short/scratch
# locations. Fall back to control directory if others
# don't exist
for path in ['/short', '/scratch', '.']:
if os.path.exists(path):
self.base = path
break

# If no lab path is set, generate a default path
if not self.basepath:
self.basepath = self.get_default_lab_path(config)
Expand All @@ -60,7 +68,7 @@ def get_default_lab_path(self, config):

# Append project name if present (NCI-specific)
default_project = os.environ.get('PROJECT', '')
default_short_path = os.path.join('/short', default_project)
default_short_path = os.path.join(self.base, default_project)
default_user = pwd.getpwuid(os.getuid()).pw_name

short_path = config.get('shortpath', default_short_path)
Expand Down
101 changes: 101 additions & 0 deletions test/common.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,39 @@

import yaml

import payu

# Namespace clash if import setup_cmd.runcmd as setup. For
# consistency use payu_ prefix for all commands
from payu.subcommands.init_cmd import runcmd as payu_init
from payu.subcommands.setup_cmd import runcmd as payu_setup_orignal
from payu.subcommands.sweep_cmd import runcmd as payu_sweep

tmpdir = Path().cwd() / Path('test') / 'tmp'
ctrldir = tmpdir / 'ctrl'
labdir = tmpdir / 'lab'
workdir = ctrldir / 'work'

config = {
'shortpath': '..',
'laboratory': 'lab',
'queue': 'normal',
'project': 'aa30',
'walltime': '0:30:00',
'ncpus': 64,
'mem': '64GB',
'jobname': 'testrun',
'model': 'test',
'exe': 'test.exe',
'input': 'testrun_1',
'manifest': {
'scaninputs': False,
'reproduce': {
'input': False,
'exe': False
}
}
}

@contextmanager
def cd(directory):
Expand Down Expand Up @@ -38,3 +71,71 @@ def get_manifests(mfdir):
with mfpath.open() as fh:
manifests[mfpath.name] = list(yaml.safe_load_all(fh))[1]
return manifests


def sweep_work(hard_sweep=False):
# Sweep workdir
with cd(ctrldir):
payu_sweep(model_type=None,
config_path=None,
hard_sweep=hard_sweep,
lab_path=str(labdir))


def payu_setup(model_type=None,
config_path=None,
lab_path=None,
force_archive=None,
reproduce=None):
"""
Wrapper around original setup command to provide default arguments
and run in ctrldir
"""
with cd(ctrldir):
payu_sweep(model_type=None,
config_path=None,
hard_sweep=False,
lab_path=str(labdir))
payu_setup_orignal(model_type,
config_path,
lab_path,
force_archive,
reproduce)


def write_config():
with (ctrldir / 'config.yaml').open('w') as file:
file.write(yaml.dump(config, default_flow_style=False))


def make_exe():
# Create a fake executable file
bindir = labdir / 'bin'
bindir.mkdir(parents=True, exist_ok=True)
exe = config['exe']
exe_size = 199
make_random_file(bindir/exe, exe_size)


def make_inputs():
# Create some fake input files
inputdir = labdir / 'input' / config['input']
inputdir.mkdir(parents=True, exist_ok=True)
for i in range(1, 4):
make_random_file(inputdir/'input_00{i}.bin'.format(i=i),
1000**2 + i)


def make_restarts():
# Create some fake restart files
restartdir = labdir / 'archive' / 'restarts'
restartdir.mkdir(parents=True, exist_ok=True)
for i in range(1, 4):
make_random_file(restartdir/'restart_00{i}.bin'.format(i=i),
5000**2 + i)


def make_all_files():
make_inputs()
make_exe()
make_restarts()
110 changes: 4 additions & 106 deletions test/test_manifest.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,118 +9,16 @@

import pdb

# Namespace clash if import setup_cmd.runcmd as setup. For
# consistency use payu_ prefix for all commands
from payu.subcommands.init_cmd import runcmd as payu_init
from payu.subcommands.setup_cmd import runcmd as payu_setup_orignal
from payu.subcommands.sweep_cmd import runcmd as payu_sweep

import payu.models.test

from common import cd, make_random_file, get_manifests

from common import tmpdir, ctrldir, labdir, workdir
from common import config, sweep_work, payu_init, payu_setup
from common import write_config
from common import make_exe, make_inputs, make_restarts, make_all_files

verbose = True

tmpdir = Path().cwd() / Path('test') / 'tmp'
ctrldir = tmpdir / 'ctrl'
labdir = tmpdir / 'lab'
workdir = ctrldir / 'work'

print(tmpdir)
print(ctrldir)
print(labdir)

config = {
'shortpath': '..',
'laboratory': 'lab',
'queue': 'normal',
'project': 'aa30',
'walltime': '0:30:00',
'ncpus': 64,
'mem': '64GB',
'jobname': 'testrun',
'model': 'test',
'exe': 'test.exe',
'input': 'testrun_1',
'manifest': {
'scaninputs': False,
'reproduce': {
'input': False,
'exe': False
}
}
}


def sweep_work(hard_sweep=False):
# Sweep workdir
with cd(ctrldir):
payu_sweep(model_type=None,
config_path=None,
hard_sweep=hard_sweep,
lab_path=str(labdir))


def payu_setup(model_type=None,
config_path=None,
lab_path=None,
force_archive=None,
reproduce=None):
"""
Wrapper around original setup command to provide default arguments
and run in ctrldir
"""
with cd(ctrldir):
payu_sweep(model_type=None,
config_path=None,
hard_sweep=False,
lab_path=str(labdir))
payu_setup_orignal(model_type,
config_path,
lab_path,
force_archive,
reproduce)


def write_config():
with (ctrldir / 'config.yaml').open('w') as file:
file.write(yaml.dump(config, default_flow_style=False))


def make_exe():
# Create a fake executable file
bindir = labdir / 'bin'
bindir.mkdir(parents=True, exist_ok=True)
exe = config['exe']
exe_size = 199
make_random_file(bindir/exe, exe_size)


def make_inputs():
# Create some fake input files
inputdir = labdir / 'input' / config['input']
inputdir.mkdir(parents=True, exist_ok=True)
for i in range(1, 4):
make_random_file(inputdir/'input_00{i}.bin'.format(i=i),
1000**2 + i)


def make_restarts():
# Create some fake restart files
restartdir = labdir / 'archive' / 'restarts'
restartdir.mkdir(parents=True, exist_ok=True)
for i in range(1, 4):
make_random_file(restartdir/'restart_00{i}.bin'.format(i=i),
5000**2 + i)


def make_all_files():
make_inputs()
make_exe()
make_restarts()


def setup_module(module):
"""
Put any test-wide setup code in here, e.g. creating test files
Expand Down

0 comments on commit cb5f653

Please sign in to comment.