diff --git a/compass/__main__.py b/compass/__main__.py index 9dd108e3bb..5d22cf83e8 100644 --- a/compass/__main__.py +++ b/compass/__main__.py @@ -2,9 +2,10 @@ import sys import argparse +import os import compass -from compass import list, setup, clean, suite, run +from compass import list, setup, clean, suite, run, cache def main(): @@ -40,11 +41,18 @@ def main(): args = parser.parse_args(sys.argv[1:2]) + # only allow the "compass cache" command if we're on Anvil or Chrysalis + allow_cache = ('COMPASS_MACHINE' in os.environ and + os.environ['COMPASS_MACHINE'] in ['anvil', 'chrysalis']) + commands = {'list': list.main, 'setup': setup.main, 'clean': clean.main, 'suite': suite.main, 'run': run.main} + if allow_cache: + commands['cache'] = cache.main + if args.command not in commands: print('Unrecognized command {}'.format(args.command)) parser.print_help() diff --git a/compass/cache.py b/compass/cache.py new file mode 100644 index 0000000000..ac2d4269ae --- /dev/null +++ b/compass/cache.py @@ -0,0 +1,135 @@ +import argparse +import json +import sys +from datetime import datetime +import os +from importlib import resources +import configparser +import shutil +import pickle + +from compass.config import add_config + + +def update_cache(step_paths, date_string=None, dry_run=False): + """ + Cache one or more compass output files for use in a cached variant of the + test case or step + + Parameters + ---------- + step_paths : list of str + The relative path of the original (uncached) steps from the base work + directory + + date_string : str, optional + The datestamp (YYMMDD) to use on the files. Default is today's date. + + dry_run : bool, optional + Whether this is a dry run (producing the json file but not copying + files to the LCRC server) + """ + if 'COMPASS_MACHINE' not in os.environ: + machine = None + invalid = True + else: + machine = os.environ['COMPASS_MACHINE'] + invalid = machine not in ['anvil', 'chrysalis'] + + if invalid: + raise ValueError('You must cache files from either Anvil or Chrysalis') + + config = configparser.ConfigParser( + interpolation=configparser.ExtendedInterpolation()) + add_config(config, 'compass.machines', '{}.cfg'.format(machine)) + + if date_string is None: + date_string = datetime.now().strftime("%y%m%d") + + # make a dictionary with MPAS cores as keys, and lists of steps as values + steps = dict() + for path in step_paths: + with open(f'{path}/step.pickle', 'rb') as handle: + _, step = pickle.load(handle) + + mpas_core = step.mpas_core.name + + if mpas_core in steps: + steps[mpas_core].append(step) + else: + steps[mpas_core] = [step] + + # now, iterate over cores and steps + for mpas_core in steps: + database_root = config.get('paths', f'{mpas_core}_database_root') + cache_root = f'{database_root}/compass_cache' + + package = f'compass.{mpas_core}' + try: + with open(f'{mpas_core}_cached_files.json') as data_file: + cached_files = json.load(data_file) + except FileNotFoundError: + # we don't have a local version of the file yet, let's see if + # there's a remote one for this MPAS core + try: + with resources.path(package, 'cached_files.json') as path: + with open(path) as data_file: + cached_files = json.load(data_file) + except FileNotFoundError: + # no cached files yet for this core + cached_files = dict() + + for step in steps[mpas_core]: + # load the step from its pickle file + + step_path = step.path + + for output in step.outputs: + output = os.path.basename(output) + out_filename = os.path.join(step_path, output) + # remove the MPAS core from the file path + target = out_filename[len(mpas_core)+1:] + path, ext = os.path.splitext(target) + target = f'{path}.{date_string}{ext}' + cached_files[out_filename] = target + + print(out_filename) + print(f' ==> {target}') + output_path = f'{cache_root}/{target}' + print(f' copy to: {output_path}') + print() + if not dry_run: + directory = os.path.dirname(output_path) + try: + os.makedirs(directory) + except FileExistsError: + pass + shutil.copyfile(out_filename, output_path) + + out_filename = f'{mpas_core}_cached_files.json' + with open(out_filename, 'w') as data_file: + json.dump(cached_files, data_file, indent=4) + + +def main(): + parser = argparse.ArgumentParser( + description='Cache the output files from one or more steps for use in ' + 'a cached variant of the step', + prog='compass cache') + parser.add_argument("-i", "--orig_steps", nargs='+', dest="orig_steps", + type=str, + help="The relative path of the original (uncached) " + "steps from the base work directory", + metavar="STEP") + parser.add_argument("-d", "--date_string", dest="date_string", type=str, + help="The datestamp (YYMMDD) to use on the files. " + "Default is today's date.", + metavar="DATE") + parser.add_argument("-r", "--dry_run", dest="dry_run", + help="Whether this is a dry run (producing the json " + "file but not copying files to the LCRC server).", + action="store_true") + + args = parser.parse_args(sys.argv[2:]) + update_cache(step_paths=args.orig_steps, date_string=args.date_string, + dry_run=args.dry_run) diff --git a/compass/mpas_core.py b/compass/mpas_core.py index 2f82cd2ae4..8911389ad4 100644 --- a/compass/mpas_core.py +++ b/compass/mpas_core.py @@ -1,3 +1,7 @@ +from importlib import resources +import json + + class MpasCore: """ The base class for housing all the tests for a given MPAS core, such as @@ -10,6 +14,11 @@ class MpasCore: test_groups : dict A dictionary of test groups for the MPAS core with their names as keys + + cached_files : dict + A dictionary that maps from output file names in test cases to cached + files in the ``compass_cache`` database for the MPAS core. These + file mappings are read in from ``cached_files.json`` in the MPAS core. """ def __init__(self, name): @@ -26,6 +35,9 @@ def __init__(self, name): # test groups are added with add_test_groups() self.test_groups = dict() + self.cached_files = dict() + self._read_cached_files() + def add_test_group(self, test_group): """ Add a test group to the MPAS core @@ -36,3 +48,16 @@ def add_test_group(self, test_group): the test group to add """ self.test_groups[test_group.name] = test_group + + def _read_cached_files(self): + """ Read in the dictionary of cached files from cached_files.json """ + + package = f'compass.{self.name}' + filename = 'cached_files.json' + try: + with resources.path(package, filename) as path: + with open(path) as data_file: + self.cached_files = json.load(data_file) + except FileNotFoundError: + # no cached files for this core + pass diff --git a/compass/ocean/cached_files.json b/compass/ocean/cached_files.json new file mode 100644 index 0000000000..9437342c67 --- /dev/null +++ b/compass/ocean/cached_files.json @@ -0,0 +1,69 @@ +{ + "ocean/global_ocean/QU240/mesh/mesh/culled_mesh.nc": "global_ocean/QU240/mesh/mesh/culled_mesh.210809.nc", + "ocean/global_ocean/QU240/mesh/mesh/culled_graph.info": "global_ocean/QU240/mesh/mesh/culled_graph.210809.info", + "ocean/global_ocean/QU240/mesh/mesh/critical_passages_mask_final.nc": "global_ocean/QU240/mesh/mesh/critical_passages_mask_final.210809.nc", + "ocean/global_ocean/QU240/PHC/init/initial_state/initial_state.nc": "global_ocean/QU240/PHC/init/initial_state/initial_state.210809.nc", + "ocean/global_ocean/QU240/PHC/init/initial_state/init_mode_forcing_data.nc": "global_ocean/QU240/PHC/init/initial_state/init_mode_forcing_data.210809.nc", + "ocean/global_ocean/QU240/PHC/init/initial_state/graph.info": "global_ocean/QU240/PHC/init/initial_state/graph.210809.info", + "ocean/global_ocean/QUwISC240/mesh/mesh/culled_mesh.nc": "global_ocean/QUwISC240/mesh/mesh/culled_mesh.210809.nc", + "ocean/global_ocean/QUwISC240/mesh/mesh/culled_graph.info": "global_ocean/QUwISC240/mesh/mesh/culled_graph.210809.info", + "ocean/global_ocean/QUwISC240/mesh/mesh/critical_passages_mask_final.nc": "global_ocean/QUwISC240/mesh/mesh/critical_passages_mask_final.210809.nc", + "ocean/global_ocean/QUwISC240/PHC/init/initial_state/initial_state.nc": "global_ocean/QUwISC240/PHC/init/initial_state/initial_state.210809.nc", + "ocean/global_ocean/QUwISC240/PHC/init/initial_state/init_mode_forcing_data.nc": "global_ocean/QUwISC240/PHC/init/initial_state/init_mode_forcing_data.210809.nc", + "ocean/global_ocean/QUwISC240/PHC/init/initial_state/graph.info": "global_ocean/QUwISC240/PHC/init/initial_state/graph.210809.info", + "ocean/global_ocean/QUwISC240/PHC/init/ssh_adjustment/adjusted_init.nc": "global_ocean/QUwISC240/PHC/init/ssh_adjustment/adjusted_init.210809.nc", + "ocean/global_ocean/EC30to60/mesh/mesh/culled_mesh.nc": "global_ocean/EC30to60/mesh/mesh/culled_mesh.210809.nc", + "ocean/global_ocean/EC30to60/mesh/mesh/culled_graph.info": "global_ocean/EC30to60/mesh/mesh/culled_graph.210809.info", + "ocean/global_ocean/EC30to60/mesh/mesh/critical_passages_mask_final.nc": "global_ocean/EC30to60/mesh/mesh/critical_passages_mask_final.210809.nc", + "ocean/global_ocean/EC30to60/PHC/init/initial_state/initial_state.nc": "global_ocean/EC30to60/PHC/init/initial_state/initial_state.210809.nc", + "ocean/global_ocean/EC30to60/PHC/init/initial_state/init_mode_forcing_data.nc": "global_ocean/EC30to60/PHC/init/initial_state/init_mode_forcing_data.210809.nc", + "ocean/global_ocean/EC30to60/PHC/init/initial_state/graph.info": "global_ocean/EC30to60/PHC/init/initial_state/graph.210809.info", + "ocean/global_ocean/WC14/mesh/mesh/culled_mesh.nc": "global_ocean/WC14/mesh/mesh/culled_mesh.210809.nc", + "ocean/global_ocean/WC14/mesh/mesh/culled_graph.info": "global_ocean/WC14/mesh/mesh/culled_graph.210809.info", + "ocean/global_ocean/WC14/mesh/mesh/critical_passages_mask_final.nc": "global_ocean/WC14/mesh/mesh/critical_passages_mask_final.210809.nc", + "ocean/global_ocean/WC14/PHC/init/initial_state/initial_state.nc": "global_ocean/WC14/PHC/init/initial_state/initial_state.210809.nc", + "ocean/global_ocean/WC14/PHC/init/initial_state/init_mode_forcing_data.nc": "global_ocean/WC14/PHC/init/initial_state/init_mode_forcing_data.210809.nc", + "ocean/global_ocean/WC14/PHC/init/initial_state/graph.info": "global_ocean/WC14/PHC/init/initial_state/graph.210809.info", + "ocean/global_ocean/ECwISC30to60/mesh/mesh/culled_mesh.nc": "global_ocean/ECwISC30to60/mesh/mesh/culled_mesh.210809.nc", + "ocean/global_ocean/ECwISC30to60/mesh/mesh/culled_graph.info": "global_ocean/ECwISC30to60/mesh/mesh/culled_graph.210809.info", + "ocean/global_ocean/ECwISC30to60/mesh/mesh/critical_passages_mask_final.nc": "global_ocean/ECwISC30to60/mesh/mesh/critical_passages_mask_final.210809.nc", + "ocean/global_ocean/ECwISC30to60/PHC/init/initial_state/initial_state.nc": "global_ocean/ECwISC30to60/PHC/init/initial_state/initial_state.210809.nc", + "ocean/global_ocean/ECwISC30to60/PHC/init/initial_state/init_mode_forcing_data.nc": "global_ocean/ECwISC30to60/PHC/init/initial_state/init_mode_forcing_data.210809.nc", + "ocean/global_ocean/ECwISC30to60/PHC/init/ssh_adjustment/adjusted_init.nc": "global_ocean/ECwISC30to60/PHC/init/ssh_adjustment/adjusted_init.210809.nc", + "ocean/global_ocean/ECwISC30to60/PHC/init/initial_state/graph.info": "global_ocean/ECwISC30to60/PHC/init/initial_state/graph.210809.info", + "ocean/global_ocean/SOwISC12to60/mesh/mesh/culled_mesh.nc": "global_ocean/SOwISC12to60/mesh/mesh/culled_mesh.210810.nc", + "ocean/global_ocean/SOwISC12to60/mesh/mesh/culled_graph.info": "global_ocean/SOwISC12to60/mesh/mesh/culled_graph.210810.info", + "ocean/global_ocean/SOwISC12to60/mesh/mesh/critical_passages_mask_final.nc": "global_ocean/SOwISC12to60/mesh/mesh/critical_passages_mask_final.210810.nc", + "ocean/global_ocean/SOwISC12to60/PHC/init/initial_state/initial_state.nc": "global_ocean/SOwISC12to60/PHC/init/initial_state/initial_state.210810.nc", + "ocean/global_ocean/SOwISC12to60/PHC/init/initial_state/init_mode_forcing_data.nc": "global_ocean/SOwISC12to60/PHC/init/initial_state/init_mode_forcing_data.210810.nc", + "ocean/global_ocean/SOwISC12to60/PHC/init/ssh_adjustment/adjusted_init.nc": "global_ocean/SOwISC12to60/PHC/init/ssh_adjustment/adjusted_init.210810.nc", + "ocean/global_ocean/SOwISC12to60/PHC/init/initial_state/graph.info": "global_ocean/SOwISC12to60/PHC/init/initial_state/graph.210810.info", + "ocean/global_convergence/cosine_bell/QU60/mesh/mesh.nc": "global_convergence/cosine_bell/QU60/mesh/mesh.210803.nc", + "ocean/global_convergence/cosine_bell/QU60/mesh/graph.info": "global_convergence/cosine_bell/QU60/mesh/graph.210803.info", + "ocean/global_convergence/cosine_bell/QU60/init/namelist.ocean": "global_convergence/cosine_bell/QU60/init/namelist.210803.ocean", + "ocean/global_convergence/cosine_bell/QU60/init/initial_state.nc": "global_convergence/cosine_bell/QU60/init/initial_state.210803.nc", + "ocean/global_convergence/cosine_bell/QU90/mesh/mesh.nc": "global_convergence/cosine_bell/QU90/mesh/mesh.210803.nc", + "ocean/global_convergence/cosine_bell/QU90/mesh/graph.info": "global_convergence/cosine_bell/QU90/mesh/graph.210803.info", + "ocean/global_convergence/cosine_bell/QU90/init/namelist.ocean": "global_convergence/cosine_bell/QU90/init/namelist.210803.ocean", + "ocean/global_convergence/cosine_bell/QU90/init/initial_state.nc": "global_convergence/cosine_bell/QU90/init/initial_state.210803.nc", + "ocean/global_convergence/cosine_bell/QU120/mesh/mesh.nc": "global_convergence/cosine_bell/QU120/mesh/mesh.210803.nc", + "ocean/global_convergence/cosine_bell/QU120/mesh/graph.info": "global_convergence/cosine_bell/QU120/mesh/graph.210803.info", + "ocean/global_convergence/cosine_bell/QU120/init/namelist.ocean": "global_convergence/cosine_bell/QU120/init/namelist.210803.ocean", + "ocean/global_convergence/cosine_bell/QU120/init/initial_state.nc": "global_convergence/cosine_bell/QU120/init/initial_state.210803.nc", + "ocean/global_convergence/cosine_bell/QU180/mesh/mesh.nc": "global_convergence/cosine_bell/QU180/mesh/mesh.210803.nc", + "ocean/global_convergence/cosine_bell/QU180/mesh/graph.info": "global_convergence/cosine_bell/QU180/mesh/graph.210803.info", + "ocean/global_convergence/cosine_bell/QU180/init/namelist.ocean": "global_convergence/cosine_bell/QU180/init/namelist.210803.ocean", + "ocean/global_convergence/cosine_bell/QU180/init/initial_state.nc": "global_convergence/cosine_bell/QU180/init/initial_state.210803.nc", + "ocean/global_convergence/cosine_bell/QU210/mesh/mesh.nc": "global_convergence/cosine_bell/QU210/mesh/mesh.210803.nc", + "ocean/global_convergence/cosine_bell/QU210/mesh/graph.info": "global_convergence/cosine_bell/QU210/mesh/graph.210803.info", + "ocean/global_convergence/cosine_bell/QU210/init/namelist.ocean": "global_convergence/cosine_bell/QU210/init/namelist.210803.ocean", + "ocean/global_convergence/cosine_bell/QU210/init/initial_state.nc": "global_convergence/cosine_bell/QU210/init/initial_state.210803.nc", + "ocean/global_convergence/cosine_bell/QU240/mesh/mesh.nc": "global_convergence/cosine_bell/QU240/mesh/mesh.210803.nc", + "ocean/global_convergence/cosine_bell/QU240/mesh/graph.info": "global_convergence/cosine_bell/QU240/mesh/graph.210803.info", + "ocean/global_convergence/cosine_bell/QU240/init/namelist.ocean": "global_convergence/cosine_bell/QU240/init/namelist.210803.ocean", + "ocean/global_convergence/cosine_bell/QU240/init/initial_state.nc": "global_convergence/cosine_bell/QU240/init/initial_state.210803.nc", + "ocean/global_convergence/cosine_bell/QU150/mesh/mesh.nc": "global_convergence/cosine_bell/QU150/mesh/mesh.210803.nc", + "ocean/global_convergence/cosine_bell/QU150/mesh/graph.info": "global_convergence/cosine_bell/QU150/mesh/graph.210803.info", + "ocean/global_convergence/cosine_bell/QU150/init/namelist.ocean": "global_convergence/cosine_bell/QU150/init/namelist.210803.ocean", + "ocean/global_convergence/cosine_bell/QU150/init/initial_state.nc": "global_convergence/cosine_bell/QU150/init/initial_state.210803.nc" +} diff --git a/compass/ocean/suites/cosine_bell_cached_init.txt b/compass/ocean/suites/cosine_bell_cached_init.txt new file mode 100644 index 0000000000..a07fcb38f1 --- /dev/null +++ b/compass/ocean/suites/cosine_bell_cached_init.txt @@ -0,0 +1,4 @@ +ocean/global_convergence/cosine_bell + cached: QU60_mesh QU60_init QU90_mesh QU90_init QU120_mesh QU120_init + cached: QU150_mesh QU150_init QU180_mesh QU180_init QU210_mesh QU210_init + cached: QU240_mesh QU240_init diff --git a/compass/ocean/suites/nightly.txt b/compass/ocean/suites/nightly.txt index bc873ea8ab..18521637c8 100644 --- a/compass/ocean/suites/nightly.txt +++ b/compass/ocean/suites/nightly.txt @@ -22,6 +22,12 @@ ocean/global_ocean/QU240/EN4_1900/performance_test ocean/global_ocean/QU240/PHC_BGC/init ocean/global_ocean/QU240/PHC_BGC/performance_test +ocean/global_ocean/QUwISC240/mesh + cached +ocean/global_ocean/QUwISC240/PHC/init + cached +ocean/global_ocean/QUwISC240/PHC/performance_test + ocean/ice_shelf_2d/5km/z-star/restart_test ocean/ice_shelf_2d/5km/z-level/restart_test diff --git a/compass/setup.py b/compass/setup.py index 537a7ec622..9ed48804ee 100644 --- a/compass/setup.py +++ b/compass/setup.py @@ -3,6 +3,7 @@ import configparser import os import pickle +import warnings from compass.mpas_cores import get_mpas_cores from compass.config import add_config, ensure_absolute_paths @@ -12,7 +13,7 @@ def setup_cases(tests=None, numbers=None, config_file=None, machine=None, work_dir=None, baseline_dir=None, mpas_model_path=None, - suite_name='custom'): + suite_name='custom', cached=None): """ Set up one or more test cases @@ -21,8 +22,10 @@ def setup_cases(tests=None, numbers=None, config_file=None, machine=None, tests : list of str, optional Relative paths for a test cases to set up - numbers : list of int, optional - Case numbers to setup, as listed from ``compass list`` + numbers : list of str, optional + Case numbers to setup, as listed from ``compass list``, optionally with + a suffix ``c`` to indicate that all steps in that test case should be + cached config_file : str, optional Configuration file with custom options for setting up and running test @@ -46,6 +49,11 @@ def setup_cases(tests=None, numbers=None, config_file=None, machine=None, The name of the test suite if tests are being set up through a test suite or ``'custom'`` if not + cached : list of list of str, optional + For each test in ``tests``, which steps (if any) should be cached, + or a list with "_all" as the first entry if all steps in the test case + should be cached + Returns ------- test_cases : dict of compass.TestCase @@ -61,6 +69,14 @@ def setup_cases(tests=None, numbers=None, config_file=None, machine=None, if tests is None and numbers is None: raise ValueError('At least one of tests or numbers is needed.') + if cached is not None: + if tests is None: + warnings.warn('Ignoring "cached" argument becasue "tests" was ' + 'not provided') + elif len(cached) != len(tests): + raise ValueError('A list of cached steps must be provided for ' + 'each test in "tests"') + if work_dir is None: work_dir = os.getcwd() work_dir = os.path.abspath(work_dir) @@ -74,20 +90,34 @@ def setup_cases(tests=None, numbers=None, config_file=None, machine=None, all_test_cases[test_case.path] = test_case test_cases = dict() + cached_steps = dict() if numbers is not None: keys = list(all_test_cases) for number in numbers: + cache_all = False + if number.endswith('c'): + cache_all = True + number = int(number[:-1]) + else: + number = int(number) + if number >= len(keys): raise ValueError('test number {} is out of range. There are ' 'only {} tests.'.format(number, len(keys))) path = keys[number] + if cache_all: + cached_steps[path] = ['_all'] + else: + cached_steps[path] = list() test_cases[path] = all_test_cases[path] if tests is not None: - for path in tests: + for index, path in enumerate(tests): if path not in all_test_cases: raise ValueError('Test case with path {} is not in ' 'test_cases'.format(path)) + if cached is not None: + cached_steps[path] = cached[index] test_cases[path] = all_test_cases[path] # get the MPAS core of the first test case. We'll assume all tests are @@ -101,7 +131,8 @@ def setup_cases(tests=None, numbers=None, config_file=None, machine=None, print('Setting up test cases:') for path, test_case in test_cases.items(): setup_case(path, test_case, config_file, machine, work_dir, - baseline_dir, mpas_model_path) + baseline_dir, mpas_model_path, + cached_steps=cached_steps[path]) test_suite = {'name': suite_name, 'test_cases': test_cases, @@ -127,7 +158,7 @@ def setup_cases(tests=None, numbers=None, config_file=None, machine=None, def setup_case(path, test_case, config_file, machine, work_dir, baseline_dir, - mpas_model_path): + mpas_model_path, cached_steps): """ Set up one or more test cases @@ -156,6 +187,10 @@ def setup_case(path, test_case, config_file, machine, work_dir, baseline_dir, mpas_model_path : str The relative or absolute path to the root of a branch where the MPAS model has been built + + cached_steps : list of str + Which steps (if any) should be cached. If all steps should be cached, + the first entry is "_all" """ print(' {}'.format(path)) @@ -227,6 +262,14 @@ def setup_case(path, test_case, config_file, machine, work_dir, baseline_dir, with open(os.path.join(test_case_dir, test_case_config), 'w') as f: config.write(f) + if len(cached_steps) > 0 and cached_steps[0] == '_all': + cached_steps = list(test_case.steps.keys()) + if len(cached_steps) > 0: + print_steps = ' '.join(cached_steps) + print(f' steps with cached outputs: {print_steps}') + for step_name in cached_steps: + test_case.steps[step_name].cached = True + # iterate over steps for step in test_case.steps.values(): # make the step directory if it doesn't exist @@ -279,10 +322,12 @@ def main(): help="Relative path for a test case to set up", metavar="PATH") parser.add_argument("-n", "--case_number", nargs='+', dest="case_num", - type=int, + type=str, help="Case number(s) to setup, as listed from " "'compass list'. Can be a space-separated" - "list of case numbers.", metavar="NUM") + "list of case numbers. A suffix 'c' indicates" + "that all steps in the test should use cached" + "outputs.", metavar="NUM") parser.add_argument("-f", "--config_file", dest="config_file", help="Configuration file for test case setup", metavar="FILE") @@ -304,16 +349,25 @@ def main(): help="The name to use for the 'custom' test suite" "containing all setup test cases.", metavar="SUITE") + parser.add_argument("--cached", dest="cached", nargs='+', + help="A list of steps in the test case supplied with" + "--test that should use cached outputs, or " + "'_all' if all steps should be cached", + metavar="STEP") args = parser.parse_args(sys.argv[2:]) + cached = None if args.test is None: tests = None else: tests = [args.test] + if args.cached is not None: + cached = [args.cached] setup_cases(tests=tests, numbers=args.case_num, config_file=args.config_file, machine=args.machine, work_dir=args.work_dir, baseline_dir=args.baseline_dir, - mpas_model_path=args.mpas_model, suite_name=args.suite_name) + mpas_model_path=args.mpas_model, suite_name=args.suite_name, + cached=cached) def _get_required_cores(test_cases): diff --git a/compass/step.py b/compass/step.py index 55874a7a59..20984f7858 100644 --- a/compass/step.py +++ b/compass/step.py @@ -76,9 +76,9 @@ class Step: time or the step will raise an exception outputs : list of str - a list of absolute paths of output files produced by this step and - available as inputs to other test cases and steps. These files must - exist after the test has run or an exception will be raised + a list of absolute paths of output files produced by this step (or + cached) and available as inputs to other test cases and steps. These + files must exist after the test has run or an exception will be raised namelist_data : dict a dictionary used internally to keep track of updates to the default @@ -111,10 +111,14 @@ class Step: log_filename : str At run time, the name of a log file where output/errors from the step are being logged, or ``None`` if output is to stdout/stderr + + cached : bool + Whether to get all of the outputs for the step from the database of + cached outputs for this MPAS core """ def __init__(self, test_case, name, subdir=None, cores=1, min_cores=1, - threads=1, max_memory=1000, max_disk=1000): + threads=1, max_memory=1000, max_disk=1000, cached=False): """ Create a new test case @@ -150,6 +154,10 @@ def __init__(self, test_case, name, subdir=None, cores=1, min_cores=1, the amount of disk space that the step is allowed to use in MB. This is currently just a placeholder for later use with task parallelism + + cached : bool, optional + Whether to get all of the outputs for the step from the database of + cached outputs for this MPAS core """ self.name = name self.test_case = test_case @@ -186,6 +194,9 @@ def __init__(self, test_case, name, subdir=None, cores=1, min_cores=1, self.logger = None self.log_filename = None + # output caching + self.cached = cached + def setup(self): """ Set up the test case in the work directory, including downloading any @@ -454,6 +465,22 @@ def process_inputs_and_outputs(self): step_dir = self.work_dir config = self.config + # process the outputs first because cached outputs will add more inputs + if self.cached: + # forget about the inputs -- we won't used them, but we will add + # the cached outputs as inputs + self.input_data = list() + for output in self.outputs: + filename = os.path.join(self.path, output) + if filename not in self.mpas_core.cached_files: + raise ValueError(f'The file {filename} has not been added ' + f'to the cache database') + target = self.mpas_core.cached_files[filename] + self.add_input_file( + filename=output, + target=target, + database='compass_cache') + inputs = [] for entry in self.input_data: filename = entry['filename'] @@ -534,6 +561,10 @@ def _generate_namelists(self): by parsing the files and dictionaries in the step's ``namelist_data``. """ + if self.cached: + # no need for namelists + return + step_work_dir = self.work_dir config = self.config @@ -570,6 +601,9 @@ def _generate_streams(self): Writes out a streams file in the work directory with new values given by parsing the files and dictionaries in the step's ``streams_data``. """ + if self.cached: + # no need for streams + return step_work_dir = self.work_dir config = self.config diff --git a/compass/suite.py b/compass/suite.py index a93b9d4c55..5d0cbc7c37 100644 --- a/compass/suite.py +++ b/compass/suite.py @@ -34,7 +34,7 @@ def setup_suite(mpas_core, suite_name, config_file=None, machine=None, directories baseline_dir : str, optional - Location of baseslines that can be compared to + Location of baselines that can be compared to mpas_model_path : str, optional The relative or absolute path to the root of a branch where the MPAS @@ -42,16 +42,13 @@ def setup_suite(mpas_core, suite_name, config_file=None, machine=None, """ text = resources.read_text('compass.{}.suites'.format(mpas_core), '{}.txt'.format(suite_name)) - tests = list() - for test in text.split('\n'): - test = test.strip() - if (len(test) > 0 and test not in tests - and not test.startswith('#')): - tests.append(test) + + tests, cached = _parse_suite(text) setup_cases(tests, config_file=config_file, machine=machine, work_dir=work_dir, baseline_dir=baseline_dir, - mpas_model_path=mpas_model_path, suite_name=suite_name) + mpas_model_path=mpas_model_path, suite_name=suite_name, + cached=cached) def clean_suite(mpas_core, suite_name, work_dir=None): @@ -75,8 +72,8 @@ def clean_suite(mpas_core, suite_name, work_dir=None): text = resources.read_text('compass.{}.suites'.format(mpas_core), '{}.txt'.format(suite_name)) - tests = [test.strip() for test in text.split('\n') if - len(test.strip()) > 0 and not test.startswith('#')] + + tests, _ = _parse_suite(text) clean_cases(tests=tests, work_dir=work_dir, suite_name=suite_name) @@ -128,3 +125,26 @@ def main(): config_file=args.config_file, machine=args.machine, work_dir=args.work_dir, baseline_dir=args.baseline_dir, mpas_model_path=args.mpas_model) + + +def _parse_suite(text): + """ Parse the text of a file defining a test suite """ + + tests = list() + cached = list() + for test in text.split('\n'): + test = test.strip() + if len(test) == 0 or test.startswith('#'): + # a blank line or comment + continue + + if test == 'cached': + cached[-1] = ['_all'] + elif test.startswith('cached:'): + steps = test[len('cached:'):].strip().split(' ') + cached[-1].extend(steps) + else: + tests.append(test) + cached.append(list()) + + return tests, cached diff --git a/compass/testcase.py b/compass/testcase.py index 2350f8b518..365329a57d 100644 --- a/compass/testcase.py +++ b/compass/testcase.py @@ -145,6 +145,9 @@ def run(self): cwd = os.getcwd() for step_name in self.steps_to_run: step = self.steps[step_name] + if step.cached: + logger.info(' * Cached: {}'.format(step_name)) + continue step.config = self.config new_log_file = self.new_step_log_file if self.log_filename is not None: diff --git a/docs/developers_guide/api.rst b/docs/developers_guide/api.rst index 0d68038a49..8a2da246bf 100644 --- a/docs/developers_guide/api.rst +++ b/docs/developers_guide/api.rst @@ -90,6 +90,17 @@ run run_step +cache +~~~~~ + +.. currentmodule:: compass.cache + +.. autosummary:: + :toctree: generated/ + + update_cache + + Base Classes ^^^^^^^^^^^^ diff --git a/docs/developers_guide/command_line.rst b/docs/developers_guide/command_line.rst index f6620ab183..e041901dea 100644 --- a/docs/developers_guide/command_line.rst +++ b/docs/developers_guide/command_line.rst @@ -276,3 +276,57 @@ Would both accomplish the same thing in this example -- skipping the over the config option. See :ref:`dev_run` for more about the underlying framework. + +.. _dev_compass_cache: + +compass cache +------------- + +``compass`` supports caching outputs from any step in a special database +called ``compass_cache`` (see :ref:`dev_step_input_download`). Files in this +database have a directory structure similar to the work directory (but without +the MPAS core subdirectory, which is redundant). The files include a date stamp +so that new revisions can be added without removing older ones (supported by +older compass versions). See :ref:`dev_step_cached_output` for more details. + +A new command, ``compass cache`` has been added to aid in updating the file +``cached_files.json`` within an MPAS core. This command is only available on +Anvil and Chrysalis, since developers can only copy files from a compass work +directory onto the LCRC server from these two machines. Developers run +``compass cache`` from the base work directory, giving the relative paths of +the step whose outputs should be cached: + +.. code-block:: bash + + compass cache -i ocean/global_ocean/QU240/mesh/mesh \ + ocean/global_ocean/QU240/PHC/init/initial_state + +This will: + +1. copy the output files from the steps directories into the appropriate + ``compass_cache`` location on the LCRC server and + +2. add these files to a local ``ocean_cached_files.json`` that can then be + copied to ``compass/ocean`` as part of a PR to add a cached version of a + step. + +The resulting ``ocean_cached_files.json`` will look something like: + +.. code-block:: json + + { + "ocean/global_ocean/QU240/mesh/mesh/culled_mesh.nc": "global_ocean/QU240/mesh/mesh/culled_mesh.210803.nc", + "ocean/global_ocean/QU240/mesh/mesh/culled_graph.info": "global_ocean/QU240/mesh/mesh/culled_graph.210803.info", + "ocean/global_ocean/QU240/mesh/mesh/critical_passages_mask_final.nc": "global_ocean/QU240/mesh/mesh/critical_passages_mask_final.210803.nc", + "ocean/global_ocean/QU240/PHC/init/initial_state/initial_state.nc": "global_ocean/QU240/PHC/init/initial_state/initial_state.210803.nc", + "ocean/global_ocean/QU240/PHC/init/initial_state/init_mode_forcing_data.nc": "global_ocean/QU240/PHC/init/initial_state/init_mode_forcing_data.210803.nc" + } + +An optional flag ``--date_string`` lets the developer set the date string to +a date they choose. The default is today's date. + +The flag ``--dry_run`` can be used to sanity check the resulting ``json`` file +and the list of files printed to stdout without actually copying the files to +the LCRC server. + +See :ref:`dev_cache` for more about the underlying framework. diff --git a/docs/developers_guide/framework.rst b/docs/developers_guide/framework.rst index f979329df2..c843cc0881 100644 --- a/docs/developers_guide/framework.rst +++ b/docs/developers_guide/framework.rst @@ -99,6 +99,19 @@ from individual steps are stored in log files ``.log`` in the test case's work directory. The results of validation (if any) are displayed in the final stage of running the test case. +.. _dev_cache: + +cache module +~~~~~~~~~~~~ + +The :py:func:`compass.cache.update_cache()` function is used by +``compass cache`` to copy step outputs to the ``compass_cache`` database on +the LCRC server and to update ``_cached_files.json`` files that +contain a mapping between these cached files and the original outputs. This +functionality enables running steps with :ref:`dev_step_cached_output`, which +can be used to skip time-consuming initialization steps for faster development +and debugging. + .. _dev_config: Config files diff --git a/docs/developers_guide/organization.rst b/docs/developers_guide/organization.rst index ac575488c9..be6253c0fe 100644 --- a/docs/developers_guide/organization.rst +++ b/docs/developers_guide/organization.rst @@ -815,13 +815,6 @@ As was the case for test cases, the base class :py:class:`compass.Step` has a large number of attributes that are useful at different stages (init, setup and run) of the step. - logger : logging.Logger - A logger for output from the step - - log_filename : str - At run time, the name of a log file where output/errors from the step - are being logged, or ``None`` if output is to stdout/stderr - Some attributes are available after calling the base class' constructor ``super().__init__()``. These include: @@ -857,6 +850,10 @@ Some attributes are available after calling the base class' constructor ``self.threads`` the number of threads the step will use +``self.cached`` + Whether to get all of the outputs for the step from the database of + cached outputs for the MPAS core that this step belongs to + Another set of attributes is not useful until ``setup()`` is called by the ``compass`` framework: @@ -897,6 +894,10 @@ framework: methods and functions that use the logger to write their output to the log file. +``self.log_filename`` + The name of a log file where output/errors from the step are being logged, + or ``None`` if output is to stdout/stderr + The inputs and outputs should not be altered but they may be used to get file names to read or write. @@ -1544,6 +1545,93 @@ in the test case's :ref:`dev_test_case_init`. The relative path in ``filename`` is with respect to the step's work directory, and is converted to an absolute path internally before the step is run. +.. _dev_step_cached_output: + +Cached output files +~~~~~~~~~~~~~~~~~~~ + +Many ``compass`` test cases and steps are expensive enough that it can become +time consuming to run full workflows to produce meshes and initial conditions +in order to test simulations. Therefore, ``compass`` provides a mechanism for +caching the outputs of each step in a database so that they can be downloaded +and symlinked rather than being computed each time. + +Cached output files are be stored in the ``compass_cache`` database within each +MPAS core's space on that LCRC server (see :ref:`dev_step_input_download`). +If the "cached" version of a step is selected, as we will describe below, each +of the test case's outputs will have a corresponding "input" file added with +the ``target`` being a cache file on the LCRC server and the ``filename`` being +the output file. ``compass`` uses the ``cached_files.json`` database to know +which cache files correspond to which step outputs. + +A developer can indicate that ``compass`` test suite includes steps with cached +outputs in two ways. First, if all steps in a test case should have cached +output, the following notation should be used: + +.. code-block:: none + + ocean/global_ocean/QU240/mesh + cached + ocean/global_ocean/QU240/PHC/init + cached + +That is, the word ``cached`` should appear after the test case on its own line. +The indentation is for visual clarity and is not required. + + +Second, ff only some steps in a test case should have cached output, they need +to be listed explicitly, as follows: + +.. code-block:: none + + ocean/global_ocean/QUwISC240/mesh + cached: mesh + ocean/global_ocean/QUwISC240/PHC/init + cached: initial_state ssh_adjustment + +The line can be indented for visual clarity, but must begin with ``cached:``, +followed by a list of steps separated by a single space. + +Similarly, a user setting up test cases has two mechanisms for specifying which +test cases and steps should have cached outputs. If all steps in a test case +should have cached outputs, the suffix ``c`` can be added to the test number: + +.. code-block:: none + + compass setup -n 90c 91c 92 ... + +In this example, test cases 90 and 91 (``mesh`` and ``init`` test cases from +the ``SOwISC12to60`` global ocean mesh, in this case) are set up with cached +outputs in all steps and 92 (``performance_test``) is not. This approach is +efficient but does not provide any control of which steps use cached outputs +and which do not. + +A much more verbose approach is required if some steps use cached outputs and +others do not within a given test case. Each test case must be set up on its +own with the ``-t`` and ``--cached`` flags as follows: + + +.. code-block:: none + + compass setup -t ocean/global_ocean/QU240/mesh --cached mesh ... + compass setup -t ocean/global_ocean/QU240/PHC/init --cached initial_state ... + ... + +Cache files should be generated by first running the test case as normal, then +running the :ref:`dev_compass_cache` command-line tool at the base of the work +directory, providing the names of the steps whose outputs should be added to +the cache. The resulting ``_cached_files.json`` should be copied +to ``compass//cached_files.json`` in a ``compass`` branch. + +Calls to ``compass cache`` must be made on Chrysalis or Anvil. If outputs were +produced on another machine, they must be transferred to one of these two +machines before calling ``compass cache``. File can be added manually to the +LCRC server and the ``cached_files.json`` databases but this is not +recommended. + +More details on cached outputs are available in the design document +:ref:`design_doc_cached_outputs`. + .. _dev_step_namelists_and_streams: Adding namelist and streams files