diff --git a/.circleci/config.yml b/.circleci/config.yml index da436cf1..7d1fb83c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -13,9 +13,9 @@ jobs: - run: | wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O conda.sh bash conda.sh -b -p ~/conda + ~/conda/bin/conda update --all ~/conda/bin/conda config --system --add channels conda-forge ~/conda/bin/conda config --system --add channels coecms - ~/conda/bin/conda update conda ~/conda/bin/conda install --yes conda-build conda-verify - run: | diff --git a/.travis.yml b/.travis.yml index 7961da47..b400e489 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,18 +1,20 @@ language: python python: - "2.7" - - "3.5" - "3.6" + - "3.7" # Disabling Torque setup until we can get this working on Travis -before_install: - - sudo apt-get install torque-server torque-client torque-mom torque-pam - - sudo ./test/setup_torque.sh +# before_install: +# - sudo apt-get install torque-server torque-client torque-mom torque-pam +# - sudo ./test/setup_torque.sh install: - pip install . before_script: - pip install -r test/requirements_test.txt script: - - PYTHONPATH=$(pwd) coverage run --source payu bin/payu + - payu list + - pylint --extension-pkg-whitelist=netCDF4 -E payu + - if [[ $TRAVIS_PYTHON_VERSION == 3.6 || $TRAVIS_PYTHON_VERSION == 3.7 ]]; then PYTHONPATH=$(pwd) coverage run --source payu -m py.test test/test_payu.py test/test_manifest.py; fi; after_success: - coverage report -m - coveralls diff --git a/README.rst b/README.rst index 414666ac..e4aa850a 100644 --- a/README.rst +++ b/README.rst @@ -1,8 +1,13 @@ Payu ==== -.. image:: https://travis-ci.org/marshallward/payu.svg?branch=master - :target: https://travis-ci.org/marshallward/payu +.. image:: https://travis-ci.org/payu-org/payu.svg?branch=master + :target: https://travis-ci.org/payu-org/payu +.. image:: https://coveralls.io/repos/github/payu-org/payu/badge.svg?branch=master + :target: https://coveralls.io/github/payu-org/payu?branch=master + + + Payu is a climate model workflow management tool for supercomputing environments. diff --git a/conda/meta.yaml b/conda/meta.yaml index 4e921bc9..c64aa2dc 100644 --- a/conda/meta.yaml +++ b/conda/meta.yaml @@ -14,6 +14,7 @@ requirements: build: - python - pbr + - setuptools run: - python - f90nml >=0.16 diff --git a/payu/cli.py b/payu/cli.py index f70651a8..6f60805b 100644 --- a/payu/cli.py +++ b/payu/cli.py @@ -20,6 +20,7 @@ import payu.envmod as envmod from payu.models import index as supported_models import payu.subcommands +from payu.scheduler.pbs import pbs_env_init # Default configuration DEFAULT_CONFIG = 'config.yaml' @@ -124,6 +125,8 @@ def set_env_vars(init_run=None, n_runs=None, lab_path=None, dir_path=None, def submit_job(pbs_script, pbs_config, pbs_vars=None): """Submit a userscript the scheduler.""" + pbs_env_init() + # Initialisation if pbs_vars is None: pbs_vars = {} diff --git a/payu/envmod.py b/payu/envmod.py index 7a7a5982..c4ac8e85 100644 --- a/payu/envmod.py +++ b/payu/envmod.py @@ -98,6 +98,7 @@ def lib_update(bin_path, lib_name): if lib_name in lib_entry: lib_path = lib_entry.split()[2] + # pylint: disable=unbalanced-tuple-unpacking mod_name, mod_version = fsops.splitpath(lib_path)[2:4] module('unload', mod_name) diff --git a/payu/experiment.py b/payu/experiment.py index e906a4be..975e1273 100644 --- a/payu/experiment.py +++ b/payu/experiment.py @@ -116,8 +116,6 @@ def __init__(self, lab, reproduce=None): self.run_id = None - pbs_env_init() - def init_models(self): self.model_name = self.config.get('model') @@ -657,7 +655,7 @@ def run(self, *user_flags): job_id = get_job_id(short=False) if job_id == '': - job_id = self.run_id[:6] + job_id = str(self.run_id)[:6] for fname in self.output_fnames: @@ -891,17 +889,16 @@ def run_userscript(self, script_cmd): # First try to interpret the argument as a full command: try: sp.check_call(shlex.split(script_cmd)) - except (EnvironmentError, sp.CalledProcessError) as exc: + except EnvironmentError as exc: # Now try to run the script explicitly - if isinstance(exc, EnvironmentError) and exc.errno == errno.ENOENT: + if exc.errno == errno.ENOENT: cmd = os.path.join(self.control_path, script_cmd) # Simplistic recursion check assert os.path.isfile(cmd) self.run_userscript(cmd) # If we get a "non-executable" error, then guess the type - elif (isinstance(exc, EnvironmentError) - and exc.errno == errno.EACCES): + elif exc.errno == errno.EACCES: # TODO: Move outside ext_cmd = {'.py': sys.executable, '.sh': '/bin/bash', @@ -919,15 +916,10 @@ def run_userscript(self, script_cmd): else: # If we can't guess the shell, then abort raise - + except sp.CalledProcessError as exc: # If the script runs but the output is bad, then warn the user - elif type(exc) == sp.CalledProcessError: - print('payu: warning: user script \'{0}\' failed (error {1}).' - ''.format(script_cmd, exc.returncode)) - - # If all else fails, raise an error - else: - raise + print('payu: warning: user script \'{0}\' failed (error {1}).' + ''.format(script_cmd, exc.returncode)) def sweep(self, hard_sweep=False): # TODO: Fix the IO race conditions! diff --git a/payu/laboratory.py b/payu/laboratory.py index e0b4ae19..d2b46bc2 100644 --- a/payu/laboratory.py +++ b/payu/laboratory.py @@ -48,6 +48,12 @@ def __init__(self, model_type=None, config_path=None, lab_path=None): self.input_basepath = os.path.join(self.basepath, 'input') self.work_path = os.path.join(self.basepath, 'work') + print("laboratory path: ", self.basepath) + print("binary path: ", self.bin_path) + print("input path: ", self.input_basepath) + print("work path: ", self.work_path) + print("archive path: ", self.archive_path) + def get_default_lab_path(self, config): """Generate a default laboratory path based on user environment.""" # Default path settings diff --git a/payu/manifest.py b/payu/manifest.py index 2cda34be..bb2e77d4 100644 --- a/payu/manifest.py +++ b/payu/manifest.py @@ -15,11 +15,10 @@ import sys import shutil import stat -from distutils.dir_util import mkpath from yamanifest.manifest import Manifest as YaManifest -from payu.fsops import make_symlink +from payu.fsops import make_symlink, mkdir_p # fast_hashes = ['nchash','binhash'] @@ -40,6 +39,7 @@ def __init__(self, path, hashes=None, ignore=None, **kwargs): self.ignore = ignore self.needsync = False + self.existing_filepaths = set() def check_fast(self, reproduce=False, **args): """ @@ -147,9 +147,8 @@ def add_filepath(self, filepath, fullpath, copy=False): if copy: self.data[filepath]['copy'] = copy - if hasattr(self, 'existing_filepaths'): - if filepath in self.existing_filepaths: - self.existing_filepaths.remove(filepath) + if filepath in self.existing_filepaths: + self.existing_filepaths.remove(filepath) return True @@ -234,20 +233,10 @@ def __init__(self, expt, reproduce): # Inherit experiment configuration self.expt = expt - self.reproduce = reproduce # Manifest control configuration self.manifest_config = self.expt.config.get('manifest', {}) - # If the run sets reproduce, default to reproduce executables. Allow - # user to specify not to reproduce executables (might not be feasible - # if executables don't match platform, or desirable if bugs existed in - # old exe) - self.reproduce_exe = ( - self.reproduce and - self.manifest_config.get('reproduce_exe', True) - ) - # Not currently supporting specifying hash functions # self.hash_functions = manifest_config.get( # 'hashfns', @@ -270,8 +259,14 @@ def __init__(self, expt, reproduce): for mf in self.manifests: self.have_manifest[mf] = False + # Set reproduce flags + self.reproduce_config = self.manifest_config.get('reproduce', {}) + self.reproduce = {} + for mf in self.manifests.keys(): + self.reproduce[mf] = self.reproduce_config.get(mf, reproduce) + # Make sure the manifests directory exists - mkpath(os.path.dirname(self.manifests['exe'].path)) + mkdir_p(os.path.dirname(self.manifests['exe'].path)) self.scaninputs = self.manifest_config.get('scaninputs', True) @@ -287,28 +282,36 @@ def __len__(self): return len(self.manifests) def setup(self): - # Check if manifest files exist - self.have_manifest['restart'] = os.path.exists( - self.manifests['restart'].path - ) - - if (os.path.exists(self.manifests['input'].path) and - not self.manifest_config.get('overwrite', False)): - # Read manifest - print('Loading input manifest: {path}' - ''.format(path=self.manifests['input'].path)) - self.manifests['input'].load() - - if len(self.manifests['input']) > 0: - self.have_manifest['input'] = True - if self.scaninputs: - self.manifests['input'].existing_filepaths = \ - set(self.manifests['input'].data.keys()) - - if self.reproduce: + if (os.path.exists(self.manifests['input'].path)): + # Always read input manifest if available + try: + print('Loading input manifest: {path}' + ''.format(path=self.manifests['input'].path)) + self.manifests['input'].load() + + if len(self.manifests['input']) > 0: + self.have_manifest['input'] = True + if self.scaninputs: + # Save existing filepath information + self.manifests['input'].existing_filepaths = \ + set(self.manifests['input'].data.keys()) + else: + # Input directories not scanned. Populate + # inputs in workdir using input manifest + print('Making input links from manifest' + '(scaninputs=False)') + self.manifests['input'].make_links() + except Exception as e: + print("Error loading input manifest: {}".format(e)) + self.manifests['input'].have_manifest = False + finally: + self.manifests['input'].have_manifest = True + + if self.reproduce['exe']: # Only load existing exe manifest if reproduce. Trivial to - # recreate and no check required for changed executable paths + # recreate and means no check required for changed + # executable paths if os.path.exists(self.manifests['exe'].path): # Read manifest print('Loading exe manifest: {}' @@ -318,6 +321,16 @@ def setup(self): if len(self.manifests['exe']) > 0: self.have_manifest['exe'] = True + # Must make links as no files will be added to the manifest + print('Making exe links') + self.manifests['exe'].make_links() + else: + self.have_manifest['exe'] = False + + if self.reproduce['restart']: + # Only load restart manifest if reproduce. Normally want to + # scan for new restarts + # Read restart manifest print('Loading restart manifest: {}' ''.format(self.have_manifest['restart'])) @@ -326,56 +339,44 @@ def setup(self): if len(self.manifests['restart']) > 0: self.have_manifest['restart'] = True - # MUST have input and restart manifests to be able to reproduce run - for mf in ['restart', 'input']: - if not self.have_manifest[mf]: - print('{} manifest cannot be empty if reproduce is True' - ''.format(mf.capitalize())) - exit(1) - - if self.reproduce_exe and not self.have_manifest['exe']: - print('Executable manifest cannot empty if reproduce and ' - 'reproduce_exe are True') - exit(1) - - # Must make links as no files will be added to the manifest - for mf in ['exe', 'restart', 'input']: - print('Making links: {}'.format(mf)) - self.manifests[mf].make_links() - for model in self.expt.models: model.have_restart_manifest = True + # Must make links as no files will be added to the manifest + print('Making restart links') + self.manifests['restart'].make_links() else: self.have_manifest['restart'] = False - if not self.scaninputs: - # If input directories not scanned then the only - # way to populate the inputs in work is to rely - # on input manifest - print('Making links from input manifest (scaninputs=False)') - self.manifests['input'].make_links() + for mf in self.manifests.keys(): + if self.reproduce[mf] and not self.have_manifest[mf]: + print('{} manifest must exist if reproduce is True' + ''.format(mf.capitalize())) + exit(1) def check_manifests(self): print("Checking exe and input manifests") - self.manifests['exe'].check_fast(reproduce=self.reproduce_exe) - if hasattr(self.manifests['input'], 'existing_filepaths'): - # Delete filepaths from input manifest - for filepath in self.manifests['input'].existing_filepaths: - print('File no longer in input directory: {file} ' - 'removing from manifest'.format(file=filepath)) - self.manifests['input'].delete(filepath) - self.manifests['input'].needsync = True - - self.manifests['input'].check_fast(reproduce=self.reproduce) - - if self.reproduce: + self.manifests['exe'].check_fast(reproduce=self.reproduce['exe']) + + if not self.reproduce['input']: + if len(self.manifests['input'].existing_filepaths) > 0: + # Delete missing filepaths from input manifest + for filepath in self.manifests['input'].existing_filepaths: + print('File no longer in input directory: {file} ' + 'removing from manifest'.format(file=filepath)) + self.manifests['input'].delete(filepath) + self.manifests['input'].needsync = True + + self.manifests['input'].check_fast(reproduce=self.reproduce['input']) + + if self.reproduce['restart']: print("Checking restart manifest") else: print("Creating restart manifest") self.manifests['restart'].needsync = True - self.manifests['restart'].check_fast(reproduce=self.reproduce) + self.manifests['restart'].check_fast( + reproduce=self.reproduce['restart']) # Write updates to version on disk for mf in self.manifests: @@ -385,7 +386,7 @@ def check_manifests(self): def copy_manifests(self, path): - mkpath(path) + mkdir_p(path) try: for mf in self.manifests: self.manifests[mf].copy(path) diff --git a/payu/models/__init__.py b/payu/models/__init__.py index c797f295..544b14b5 100644 --- a/payu/models/__init__.py +++ b/payu/models/__init__.py @@ -9,6 +9,7 @@ from payu.models.mom6 import Mom6 from payu.models.nemo import Nemo from payu.models.oasis import Oasis +from payu.models.test import Test from payu.models.um import UnifiedModel from payu.models.ww3 import WW3 from payu.models.qgcm import Qgcm @@ -28,6 +29,7 @@ 'mom': Mom, 'nemo': Nemo, 'oasis': Oasis, + 'test': Test, 'um': UnifiedModel, 'ww3': WW3, 'mom6': Mom6, diff --git a/payu/models/mitgcm.py b/payu/models/mitgcm.py index 90eee76e..51f8502a 100644 --- a/payu/models/mitgcm.py +++ b/payu/models/mitgcm.py @@ -78,7 +78,7 @@ def setup(self): if self.prior_restart_path and not self.expt.repeat_run: # Determine total number of timesteps since initialisation core_restarts = [f for f in os.listdir(self.prior_restart_path) - if f.startswith('pickup.')] + if f.startswith('pickup.')] try: # NOTE: Use the most recent, in case of multiple restarts n_iter0 = max([int(f.split('.')[1]) for f in core_restarts]) @@ -125,7 +125,8 @@ def setup(self): # Assume n_timesteps and dt set correctly pass - if t_start is None or (self.prior_restart_path and not self.expt.repeat_run): + if t_start is None or (self.prior_restart_path + and not self.expt.repeat_run): # Look for a restart file from a previous run if os.path.exists(restart_calendar_path): with open(restart_calendar_path, 'r') as restart_file: @@ -139,23 +140,23 @@ def setup(self): # Check if deltat has changed if n_iter0 != t_start // dt: - n_iter0_previous = n_iter0 - # Specify a pickup suffix using previous niter0 - data_nml['parm03']['pickupsuff'] = '{:010d}'.format(n_iter0_previous) + data_nml['parm03']['pickupsuff'] = '{:010d}'.format(n_iter0) + + n_iter0_previous = n_iter0 n_iter0 = t_start // dt if n_iter0 * dt != t_start: - print('payu : error: Timestep changed to {dt}. New timestep not' - ' integer multiple of start time: {start}'.format(dt=dt, - start=t_start)) + print('payu : error: Timestep changed to {dt}. New timestep ' + 'not integer multiple of start time: ' + '{start}'.format(dt=dt, start=t_start)) sys.exit(1) if n_iter0 + n_timesteps == n_iter0_previous: - print('payu : error: Timestep changed to {dt}. Timestep at end' - ' identical to previous pickups: {niter}'.format(dt=dt, - niter=(n_iter0 + n_timesteps))) + print('payu : error: Timestep changed to {dt}. ' + 'Timestep at end identical to previous pickups: ' + '{niter}'.format(dt=dt, niter=(n_iter0 + n_timesteps))) print('This would overwrite previous pickups') sys.exit(1) @@ -180,8 +181,8 @@ def setup(self): # NOTE: Consider permitting pchkpt_freq < dt * n_timesteps if t_end % pchkpt_freq != 0: - # Terrible hack for when we change dt, the pickup frequency no longer - # make sense, so have to set it to the total runtime + # Terrible hack for when we change dt, the pickup frequency + # no longer make sense, so have to set it to the total runtime data_nml['parm03']['pchkptfreq'] = t_end else: data_nml['parm03']['pchkptfreq'] = pchkpt_freq @@ -209,9 +210,8 @@ def setup(self): 'mnc_outdir_date': True, 'monitor_mnc': True } - data_mnc_nml = {'mnc_01': mnc_01_grp} - - nml_parser.write(data_mnc_nml, data_mnc_path) + data_mnc_nml = f90nml.Namelist(mnc_01=mnc_01_grp) + data_mnc_nml.write(data_mnc_path) else: raise @@ -225,8 +225,8 @@ def archive(self): # Save model time to restart next run with open(os.path.join(self.restart_path, self.restart_calendar_file), 'w') as restart_file: - restart_file.write(yaml.dump({'endtime': data_nml['parm03']['endTime']}, - default_flow_style=False)) + restart = {'endtime': data_nml['parm03']['endTime']}, + restart_file.write(yaml.dump(restart, default_flow_style=False)) # Remove symbolic links to input or pickup files: for f in os.listdir(self.work_path): diff --git a/payu/models/model.py b/payu/models/model.py index 5370d0d1..e4e3b337 100644 --- a/payu/models/model.py +++ b/payu/models/model.py @@ -190,8 +190,9 @@ def set_model_output_paths(self): def get_prior_restart_files(self): try: - return [f for f in os.listdir(self.prior_restart_path) - if os.path.isfile(os.path.join(self.prior_restart_path, f))] + respath = self.prior_restart_path + return [f for f in os.listdir(respath) + if os.path.isfile(os.path.join(respath, f))] except Exception as e: print("No prior restart files found: {error}".format(error=str(e))) return [] @@ -235,7 +236,7 @@ def setup(self): ) # Add input files to manifest if we don't already have a populated - # input manifest, or we specify scan_inputs is True (default) + # input manifest, or we specify scaninputs is True (default) if (not self.expt.manifest.have_manifest['input'] or self.expt.manifest.scaninputs): for input_path in self.input_paths: diff --git a/payu/models/mom.py b/payu/models/mom.py index 83fa932d..d06e9604 100644 --- a/payu/models/mom.py +++ b/payu/models/mom.py @@ -137,6 +137,9 @@ def set_timestep(self, timestep): def create_mask_table(self, input_nml): import netCDF4 + # Disable E1136 which is tripped below when accessing grid_vars + # pylint: disable=unsubscriptable-object + # Get the grid spec path grid_spec_fname = 'grid_spec.nc' for input_dir in self.input_paths: @@ -163,6 +166,8 @@ def create_mask_table(self, input_nml): ocn_topog_path = os.path.join(input_dir, ocn_topog_fname) break + # pylint: enable=unsubscriptable-object + grid_spec_nc.close() check_mask = os.path.join(self.expt.lab.bin_path, 'check_mask') diff --git a/payu/models/test.py b/payu/models/test.py new file mode 100644 index 00000000..a6c85511 --- /dev/null +++ b/payu/models/test.py @@ -0,0 +1,31 @@ +"""Test driver interface + +:copyright: Copyright 2019 Marshall Ward, see AUTHORS for details +:license: Apache License, Version 2.0, see LICENSE for details +""" +import os +import shlex +import shutil +import subprocess + +from payu.models.model import Model + +config_files = [ + 'data', + 'diag', + 'input.nml' + ] + + +class Test(Model): + + def __init__(self, expt, name, config): + + # payu initialisation + super(Test, self).__init__(expt, name, config) + + # Model-specific configuration + self.model_type = 'test' + self.default_exec = 'test.exe' + + self.config_files = config_files diff --git a/payu/scheduler/pbs.py b/payu/scheduler/pbs.py index 2c19d917..33b4564d 100644 --- a/payu/scheduler/pbs.py +++ b/payu/scheduler/pbs.py @@ -16,6 +16,7 @@ from tenacity import retry, stop_after_delay + def get_job_id(short=True): """ Return PBS job id @@ -29,6 +30,7 @@ def get_job_id(short=True): return(jobid) + def get_job_info(): """ Get information about the job from the PBS server @@ -41,7 +43,7 @@ def get_job_info(): info = get_qstat_info('-ft {0}'.format(jobid), 'Job Id:') if info is not None: - # Select the dict for this job (there should only be one + # Select the dict for this job (there should only be one # entry in any case) info = info['Job Id: {}'.format(jobid)] diff --git a/payu/subcommands/profile_cmd.py b/payu/subcommands/profile_cmd.py index 8e660a47..bd86c2e6 100644 --- a/payu/subcommands/profile_cmd.py +++ b/payu/subcommands/profile_cmd.py @@ -8,6 +8,7 @@ from payu import cli from payu.experiment import Experiment from payu.laboratory import Laboratory +from payu import fsops import payu.subcommands.args as args title = 'profile' @@ -19,7 +20,7 @@ def runcmd(model_type, config_path, init_run, n_runs, lab_path): - pbs_config = cli.get_config(config_path) + pbs_config = fsops.read_config(config_path) pbs_vars = cli.set_env_vars(init_run, n_runs, lab_path) pbs_config['queue'] = pbs_config.get('profile_queue', 'normal') diff --git a/payu/subcommands/run_cmd.py b/payu/subcommands/run_cmd.py index 1fa97704..ca559dac 100644 --- a/payu/subcommands/run_cmd.py +++ b/payu/subcommands/run_cmd.py @@ -41,7 +41,7 @@ def runcmd(model_type, config_path, init_run, n_runs, lab_path, reproduce): # TODO: Is control_path defined at this stage? mask_table_fname = None for fname in os.listdir(os.curdir): - if f.startswith('mask_table'): + if fname.startswith('mask_table'): mask_table_fname = fname # TODO TODO diff --git a/setup.py b/setup.py index f622bc2e..d8d2ef85 100644 --- a/setup.py +++ b/setup.py @@ -36,6 +36,7 @@ 'yamanifest', 'dateutil', 'tenacity', + 'scipy', ], install_requires=[ 'f90nml >= 0.16', @@ -45,6 +46,10 @@ 'python-dateutil', 'tenacity', ], + tests_require=[ + 'pytest', + 'pylint', + ], entry_points={ 'console_scripts': [ 'payu = payu.cli:parse', diff --git a/test/common.py b/test/common.py new file mode 100644 index 00000000..9d579095 --- /dev/null +++ b/test/common.py @@ -0,0 +1,40 @@ +from contextlib import contextmanager +import os +from pathlib import Path + +import yaml + + +@contextmanager +def cd(directory): + """ + Context manager to change into a directory and + change back to original directory when complete + """ + old_dir = Path.cwd() + os.chdir(directory) + try: + yield + finally: + os.chdir(old_dir) + + +def make_random_file(filename, size=1000**2): + """ + Make a file of specified size filled with some + random content + """ + with open(filename, 'wb') as fout: + fout.write(os.urandom(size)) + + +def get_manifests(mfdir): + """ + Read in all manifests and return as a dict + """ + manifests = {} + for mf in ['exe', 'input', 'restart']: + mfpath = Path(mfdir)/"{}.yaml".format(mf) + with mfpath.open() as fh: + manifests[mfpath.name] = list(yaml.safe_load_all(fh))[1] + return manifests diff --git a/test/requirements_test.txt b/test/requirements_test.txt index 79e3a4ee..6b554902 100644 --- a/test/requirements_test.txt +++ b/test/requirements_test.txt @@ -1,3 +1,7 @@ coverage == 3.7.1 ; python_version == '3.2' coverage ; python_version != '3.2' coveralls +pytest +pylint +mnctools +scipy \ No newline at end of file diff --git a/test/test_manifest.py b/test/test_manifest.py new file mode 100644 index 00000000..7b67c1dd --- /dev/null +++ b/test/test_manifest.py @@ -0,0 +1,546 @@ +import os +from pathlib import Path +import shutil + +import pytest +import yaml + +import payu + +import pdb + +# Namespace clash if import setup_cmd.runcmd as setup. For +# consistency use payu_ prefix for all commands +from payu.subcommands.init_cmd import runcmd as payu_init +from payu.subcommands.setup_cmd import runcmd as payu_setup_orignal +from payu.subcommands.sweep_cmd import runcmd as payu_sweep + +import payu.models.test + +from common import cd, make_random_file, get_manifests + + +verbose = True + +tmpdir = Path().cwd() / Path('test') / 'tmp' +ctrldir = tmpdir / 'ctrl' +labdir = tmpdir / 'lab' +workdir = ctrldir / 'work' + +print(tmpdir) +print(ctrldir) +print(labdir) + +config = { + 'shortpath': '..', + 'laboratory': 'lab', + 'queue': 'normal', + 'project': 'aa30', + 'walltime': '0:30:00', + 'ncpus': 64, + 'mem': '64GB', + 'jobname': 'testrun', + 'model': 'test', + 'exe': 'test.exe', + 'input': 'testrun_1', + 'manifest': { + 'scaninputs': False, + 'reproduce': { + 'input': False, + 'exe': False + } + } + } + + +def sweep_work(hard_sweep=False): + # Sweep workdir + with cd(ctrldir): + payu_sweep(model_type=None, + config_path=None, + hard_sweep=hard_sweep, + lab_path=str(labdir)) + + +def payu_setup(model_type=None, + config_path=None, + lab_path=None, + force_archive=None, + reproduce=None): + """ + Wrapper around original setup command to provide default arguments + and run in ctrldir + """ + with cd(ctrldir): + payu_sweep(model_type=None, + config_path=None, + hard_sweep=False, + lab_path=str(labdir)) + payu_setup_orignal(model_type, + config_path, + lab_path, + force_archive, + reproduce) + + +def write_config(): + with (ctrldir / 'config.yaml').open('w') as file: + file.write(yaml.dump(config, default_flow_style=False)) + + +def make_exe(): + # Create a fake executable file + bindir = labdir / 'bin' + bindir.mkdir(parents=True, exist_ok=True) + exe = config['exe'] + exe_size = 199 + make_random_file(bindir/exe, exe_size) + + +def make_inputs(): + # Create some fake input files + inputdir = labdir / 'input' / config['input'] + inputdir.mkdir(parents=True, exist_ok=True) + for i in range(1, 4): + make_random_file(inputdir/'input_00{i}.bin'.format(i=i), + 1000**2 + i) + + +def make_restarts(): + # Create some fake restart files + restartdir = labdir / 'archive' / 'restarts' + restartdir.mkdir(parents=True, exist_ok=True) + for i in range(1, 4): + make_random_file(restartdir/'restart_00{i}.bin'.format(i=i), + 5000**2 + i) + + +def make_all_files(): + make_inputs() + make_exe() + make_restarts() + + +def setup_module(module): + """ + Put any test-wide setup code in here, e.g. creating test files + """ + if verbose: + print("setup_module module:%s" % module.__name__) + + # Should be taken care of by teardown, in case remnants lying around + try: + shutil.rmtree(tmpdir) + except FileNotFoundError: + pass + + try: + tmpdir.mkdir() + labdir.mkdir() + ctrldir.mkdir() + except Exception as e: + print(e) + + write_config() + + +def teardown_module(module): + """ + Put any test-wide teardown code in here, e.g. removing test outputs + """ + if verbose: + print("teardown_module module:%s" % module.__name__) + + try: + # shutil.rmtree(tmpdir) + print('removing tmp') + except Exception as e: + print(e) + +# These are integration tests. They have an undesirable dependence on each +# other. It would be possible to make them independent, but then they'd +# be reproducing previous "tests", like init. So this design is deliberate +# but compromised. It means when running an error in one test can cascade +# and cause other tests to fail. +# +# Unfortunate but there you go. + + +def test_init(): + + # Initialise a payu laboratory + with cd(ctrldir): + payu_init(None, None, str(labdir)) + + # Check all the correct directories have been created + for subdir in ['bin', 'input', 'archive', 'codebase']: + assert((labdir / subdir).is_dir()) + + +def test_setup(): + + # Create some input and executable files + make_inputs() + make_exe() + + bindir = labdir / 'bin' + exe = config['exe'] + + config_files = payu.models.test.config_files + for file in config_files: + make_random_file(ctrldir/file, 29) + + # Run setup + payu_setup(lab_path=str(labdir)) + + assert(workdir.is_symlink()) + assert(workdir.is_dir()) + assert((workdir/exe).resolve() == (bindir/exe).resolve()) + workdirfull = workdir.resolve() + + for f in config_files + ['config.yaml']: + assert((workdir/f).is_file()) + + for i in range(1, 4): + assert((workdir/'input_00{i}.bin'.format(i=i)).stat().st_size + == 1000**2 + i) + + manifests = get_manifests(ctrldir/'manifests') + for mfpath in manifests: + assert((ctrldir/'manifests'/mfpath).is_file()) + + # Check manifest in work directory is the same as control directory + assert(manifests == get_manifests(workdir/'manifests')) + + # Sweep workdir and recreate + sweep_work() + + assert(not workdir.is_dir()) + assert(not workdirfull.is_dir()) + + payu_setup(lab_path=str(labdir)) + + assert(manifests == get_manifests(workdir/'manifests')) + + +def test_setup_restartdir(): + + restartdir = labdir / 'archive' / 'restarts' + + # Set a restart directory in config + config['restart'] = str(restartdir) + write_config() + + make_restarts() + + manifests = get_manifests(ctrldir/'manifests') + payu_setup(lab_path=str(labdir)) + + # Manifests should not match, as have added restarts + assert(not manifests == get_manifests(ctrldir/'manifests')) + + +def test_exe_reproduce(): + + # Set reproduce exe to True + config['manifest']['reproduce']['exe'] = True + write_config() + manifests = get_manifests(ctrldir/'manifests') + + # Run setup with unchanged exe but reproduce exe set to True. + # Should run without error + payu_setup(lab_path=str(labdir)) + + assert(manifests == get_manifests(ctrldir/'manifests')) + + bindir = labdir / 'bin' + exe = config['exe'] + + # Update the modification time of the executable, should run ok + (bindir/exe).touch() + + # Run setup with changed exe but reproduce exe set to False + payu_setup(lab_path=str(labdir)) + + # Manifests will have changed as fasthash is altered + assert(not manifests == get_manifests(ctrldir/'manifests')) + + # Reset manifests "truth" + manifests = get_manifests(ctrldir/'manifests') + + # Delete exe path from config, should get it from manifest + del(config['exe']) + write_config() + + # Run setup with changed exe but reproduce exe set to False + payu_setup(lab_path=str(labdir)) + + # Manifests will not have changed + assert(manifests == get_manifests(ctrldir/'manifests')) + assert((workdir/exe).resolve() == (bindir/exe).resolve()) + + # Reinstate exe path + config['exe'] = exe + + # Recreate fake executable file + make_exe() + + # Run setup again, which should raise an error due to changed executable + with pytest.raises(SystemExit) as pytest_wrapped_e: + # Run setup with unchanged exe but reproduce exe set to True + payu_setup(lab_path=str(labdir)) + + assert pytest_wrapped_e.type == SystemExit + assert pytest_wrapped_e.value.code == 1 + + # Change reproduce exe back to False + config['manifest']['reproduce']['exe'] = False + write_config() + + # Run setup with changed exe but reproduce exe set to False + payu_setup(lab_path=str(labdir)) + + # Check manifests have changed as expected + assert(not manifests == get_manifests(ctrldir/'manifests')) + + +def test_input_reproduce(): + + inputdir = labdir / 'input' / config['input'] + inputdir.mkdir(parents=True, exist_ok=True) + + # Set reproduce input to True + config['manifest']['reproduce']['exe'] = False + config['manifest']['reproduce']['input'] = True + write_config() + manifests = get_manifests(ctrldir/'manifests') + + # Run setup with unchanged input reproduce input set to True + # to make sure works with no changes + payu_setup(lab_path=str(labdir)) + assert(manifests == get_manifests(ctrldir/'manifests')) + + # Delete input directory from config, should still work from + # manifest with input reproduce True + input = config['input'] + write_config() + del(config['input']) + + # Run setup, should work + payu_setup(lab_path=str(labdir)) + + assert(manifests == get_manifests(ctrldir/'manifests')) + + # Update modification times for input files + for i in range(1, 4): + (inputdir/'input_00{i}.bin'.format(i=i)).touch() + + # Run setup, should work as only fasthash will differ, code then + # checks full hash and updates fasthash if fullhash matches + payu_setup(lab_path=str(labdir)) + + # Manifests should no longer match as fasthashes have been updated + assert(not manifests == get_manifests(ctrldir/'manifests')) + + # Reset manifest "truth" + manifests = get_manifests(ctrldir/'manifests') + + # Re-create input files. Have to set input path for this purpose + # but not written to config.yaml, so doesn't affect payu commands + config['input'] = input + make_inputs() + del(config['input']) + + # Run setup again, which should raise an error due to changed inputs + with pytest.raises(SystemExit) as pytest_wrapped_e: + payu_setup(lab_path=str(labdir)) + + assert pytest_wrapped_e.type == SystemExit + assert pytest_wrapped_e.value.code == 1 + + # Change reproduce input back to False + config['manifest']['reproduce']['input'] = False + write_config() + + # Run setup with changed inputs but reproduce input set to False + payu_setup(lab_path=str(labdir)) + + # Check manifests have changed as expected and input files + # linked in work + assert(not manifests == get_manifests(ctrldir/'manifests')) + for i in range(1, 4): + assert((workdir/'input_00{i}.bin'.format(i=i)).is_file()) + + # Reset manifest "truth" + manifests = get_manifests(ctrldir/'manifests') + + # Delete input manifest + (ctrldir/'manifests'/'input.yaml').unlink() + + # Setup with no input dir and no manifest. Should work ok + payu_setup(lab_path=str(labdir)) + + # Check there are no linked inputs + for i in range(1, 4): + assert(not (workdir/'input_00{i}.bin'.format(i=i)).is_file()) + + # Set input path back and recreate input manifest + config['input'] = input + write_config() + payu_setup(lab_path=str(labdir)) + + +def test_input_scaninputs(): + + inputdir = labdir / 'input' / config['input'] + inputdir.mkdir(parents=True, exist_ok=True) + + # Set scaninputs input to True + config['manifest']['scaninputs'] = True + write_config() + + # Run setup with unchanged input + payu_setup(lab_path=str(labdir)) + manifests = get_manifests(ctrldir/'manifests') + + # Set scaninputs input to False + config['manifest']['scaninputs'] = False + write_config() + + # Run setup, should work and manifests unchanged + payu_setup(lab_path=str(labdir)) + assert(manifests == get_manifests(ctrldir/'manifests')) + + # Update modification times for input files + for i in range(1, 4): + (inputdir/'input_00{i}.bin'.format(i=i)).touch() + + # Run setup, should work as only fasthash will differ, code then + # checks full hash and updates fasthash if fullhash matches + payu_setup(lab_path=str(labdir)) + + # Manifests should no longer match as fasthashes have been updated + assert(not manifests == get_manifests(ctrldir/'manifests')) + + # Reset manifest "truth" + manifests = get_manifests(ctrldir/'manifests') + + # Re-create input files + make_inputs() + + # Run setup again. Should be fine, but manifests changed + payu_setup(lab_path=str(labdir)) + assert(not manifests == get_manifests(ctrldir/'manifests')) + + # Reset manifest "truth" + manifests = get_manifests(ctrldir/'manifests') + + # Make a new input file + (inputdir/'lala').touch() + + # Run setup again. Should be fine, manifests unchanged as + # scaninputs=False + payu_setup(lab_path=str(labdir)) + assert(manifests == get_manifests(ctrldir/'manifests')) + + # Set scaninputs input to True + config['manifest']['scaninputs'] = True + write_config() + + # Run setup again. Should be fine, but manifests changed now + # as scaninputs=False + payu_setup(lab_path=str(labdir)) + assert(not manifests == get_manifests(ctrldir/'manifests')) + assert((workdir/'lala').is_file()) + + # Delete silly input file + (inputdir/'lala').unlink() + + # Re-run after removing silly input file + payu_setup(lab_path=str(labdir)) + + # Reset manifest "truth" + manifests = get_manifests(ctrldir/'manifests') + + +def test_restart_reproduce(): + + # Set reproduce restart to True + config['manifest']['reproduce']['input'] = False + config['manifest']['reproduce']['restart'] = True + del(config['restart']) + write_config() + manifests = get_manifests(ctrldir/'manifests') + + # Run setup with unchanged restarts + payu_setup(lab_path=str(labdir)) + assert(manifests == get_manifests(ctrldir/'manifests')) + + restartdir = labdir / 'archive' / 'restarts' + # Change modification times on restarts + for i in range(1, 4): + (restartdir/'restart_00{i}.bin'.format(i=i)).touch() + + # Run setup with touched restarts, should work with modified + # manifest + payu_setup(lab_path=str(labdir)) + + # Manifests should have changed + assert(not manifests == get_manifests(ctrldir/'manifests')) + + # Reset manifest "truth" + manifests = get_manifests(ctrldir/'manifests') + + # Modify restart files + make_restarts() + + # Run setup again, which should raise an error due to changed restarts + with pytest.raises(SystemExit) as pytest_wrapped_e: + # Run setup with unchanged exe but reproduce exe set to True + payu_setup(lab_path=str(labdir)) + + # Set reproduce restart to False + config['manifest']['reproduce']['restart'] = False + write_config() + + # Run setup with modified restarts reproduce set to False + payu_setup(lab_path=str(labdir)) + + # Manifests should have changed + assert(not manifests == get_manifests(ctrldir/'manifests')) + + +def test_all_reproduce(): + + # Remove reproduce options from config + del(config['manifest']['reproduce']) + write_config() + + # Run setup + payu_setup(lab_path=str(labdir)) + + manifests = get_manifests(ctrldir/'manifests') + + make_all_files() + + # Run setup with reproduce=True, which should raise an error as + # all files changed + with pytest.raises(SystemExit) as pytest_wrapped_e: + # Run setup with unchanged exe but reproduce exe set to True + payu_setup(lab_path=str(labdir), reproduce=True) + + # Run setup + payu_setup(lab_path=str(labdir)) + + # Manifests should have changed + assert(not manifests == get_manifests(ctrldir/'manifests')) + + +def test_hard_sweep(): + + # Sweep workdir + sweep_work(hard_sweep=True) + + # Check all the correct directories have been removed + assert(not (labdir / 'archive' / 'ctrl').is_dir()) + assert(not (labdir / 'work' / 'ctrl').is_dir()) diff --git a/test/test_payu.py b/test/test_payu.py index 579d3db0..1b497a57 100644 --- a/test/test_payu.py +++ b/test/test_payu.py @@ -17,6 +17,7 @@ import payu.fsops import payu.laboratory + class Test(unittest.TestCase): def setUp(self): @@ -41,7 +42,7 @@ def test_mkdir_p(self): os.rmdir(tmp_dir) def test_read_config(self): - config_path = 'config_mom5.yaml' + config_path = os.path.join('test', 'config_mom5.yaml') config = payu.fsops.read_config(config_path) # Raise a non-ENOENT error (e.g. EACCES) @@ -115,5 +116,6 @@ def test_lab_new(self): lab = payu.laboratory.Laboratory('model') sys.stdout = sys.__stdout__ + if __name__ == '__main__': unittest.main()