diff --git a/nightly_testing/example_meto_configs.yaml b/nightly_testing/example_meto_configs.yaml index c47ec13..f1d28ee 100644 --- a/nightly_testing/example_meto_configs.yaml +++ b/nightly_testing/example_meto_configs.yaml @@ -44,7 +44,7 @@ lfric_apps_excd_heads_nightly: vars: - HOUSEKEEPING=false - USE_EXCD=true - time_launch: 04:45 + time_launch: 04:30 time_clean: 04:50 period: nightly_all @@ -154,13 +154,25 @@ um_heads_weekly-portio2b: groups: fcm_make_portio2b,compiler_warning_check revisions: set vars: - - USE_EXAB=true - HOUSEKEEPING=false - PREBUILDS=false - time_launch: 05:30 + time_launch: 06:00 time_clean: 05:45 period: weekly + +um_exz_heads_weekly: + repo: um + groups: ex1a_cce_n48_ga8_amip_2day + revisions: heads + vars: + - HOUSEKEEPING=false + - PREBUILDS=false + - USE_EXZ=true + time_launch: 03:10 + time_clean: 05:20 + period: weekly + ################### # Nightly Testing # ################### @@ -225,7 +237,7 @@ um_set-revisions_next-cylc_nightly: jules_nightly: repo: jules - groups: all + groups: all,fab vars: - USE_EXAB=true - HOUSEKEEPING=false @@ -240,6 +252,6 @@ jules_excd_nightly: vars: - HOUSEKEEPING=false - USE_EXCD=true - time_launch: 05:55 + time_launch: 06:00 time_clean: 06:30 period: nightly_all diff --git a/nightly_testing/generate_test_suite_cron.py b/nightly_testing/generate_test_suite_cron.py index 5c940c9..d2ecd70 100755 --- a/nightly_testing/generate_test_suite_cron.py +++ b/nightly_testing/generate_test_suite_cron.py @@ -27,47 +27,35 @@ * revisions: heads to use the HoT for sub-repos, set to use the set revisions * vars: strings that follow the -S command on the command line * monitoring: Boolean, whether to run the monitoring script on this suite -* cylc_version: Can be any string beginning 7 or 8 that is a valid cylc install +* cylc_version: Can be any string beginning 8 that is a valid cylc install at the site, such that `export CYLC_VERSION=` works. """ import argparse import os -import re import subprocess import sys - import yaml -DEFAULT_CYLC_VERSION = "8" DEPENDENCIES = { "lfric_apps": [ "casim", "jules", - "lfric", + "lfric_core", "shumlib", "socrates", "ukca", "um", ], "um": ["casim", "jules", "mule", "shumlib", "socrates", "ukca"], - "lfric": [], + "lfric_core": [], "jules": [], "ukca": [], } -CYLC_DIFFS = { - "7": { - "name": "--name=", - "clean": "rose suite-clean", - }, - "8": { - "name": "--workflow-name=", - "clean": "cylc clean --timeout=7200", - }, -} -WC_DIR = os.path.join(os.environ["TMPDIR"], os.environ["USER"]) +CLONE_DIR = os.path.join(os.environ["TMPDIR"], os.environ["USER"]) +MIRROR_PATH = "/data/users/gitassist/git_mirrors/" UMDIR = os.environ["UMDIR"] PROFILE = ". /etc/profile" DATE_BASE = "date +\\%Y-\\%m-\\%d" @@ -83,63 +71,23 @@ def run_command(command): ) -def major_cylc_version(cylc_version): - """ - Return the major version of cylc being requested by cylc_version - Expected to be 7 or 8 - """ - return re.split("[._-]", cylc_version)[0] - - -def join_checkout_commands(repos, dir_wc): +def create_git_clone_cron(repo): """ - Join commands that checkout new repos + Return a string of cron commands to get a git clone from the gitassist mirror + Runs at 23:30 each day before all other tasks """ - command = "" - for repo in repos: - wc_path = os.path.join(dir_wc, "wc_" + repo) - command += f"fcm co -q --force fcm:{repo}.xm_tr@HEAD {wc_path} ; " - return command - - -def fetch_working_copy_cron(): - """ - Cleanup and then re-checkout working copies for each of the repos used - Create an lfric_apps heads working copy - Runs at 23:30, before all other tasks - """ - - command = "# Checkout Working Copies - every day at 23:30 #" - l = len(command) - command = f"{l*'#'}\n{command}\n{l*'#'}\n30 23 * * * {PROFILE} ; " - command += f"rm -rf {os.path.join(WC_DIR, 'wc_*')} ; " - command += join_checkout_commands(DEPENDENCIES.keys(), WC_DIR) - command += lfric_heads_sed(os.path.join(WC_DIR, "wc_lfric_apps")) + clone_path = os.path.join(CLONE_DIR, f"clone_{repo}") + repo_mirror = os.path.join(MIRROR_PATH, "MetOffice", f"{repo}.git") + command = f"# Clone {repo} - every day at 23:30 #" + length = len(command) + command = f"{length*'#'}\n{command}\n{length*'#'}\n30 23 * * * {PROFILE} ; " + command += f"rm -rf {clone_path} ; " + command += f"git clone {repo_mirror} {clone_path}" return command + "\n\n\n" -def lfric_heads_sed(wc_path): - """ - Add 2 sed commands to setup dependencies.sh for heads testing - One command replaces all revisions with HEAD - The other removes all sources - in the event a branch/working copy is left in at - commit, this means that the Heads testing should still run. The set-revisions - testing would show the wrong source. - As this edits the working copy it copies the original copy with _heads added - and returns this new wc path - """ - - wc_path_new = wc_path + "_heads" - dep_path = os.path.join(wc_path_new, "dependencies.sh") - - rstr = f"cp -rf {wc_path} {wc_path_new} ; " - rstr += f'sed -i -e "s/^\\(export .*_rev=\\).*/\\1HEAD/" {dep_path} ; ' - rstr += f'sed -i -e "s/^\\(export .*_sources=\\).*/\\1/" {dep_path} ; ' - return rstr - - def generate_cron_timing_str(suite, mode): """ Return a string with the cron timing info included but no commands @@ -229,7 +177,7 @@ def generate_clean_commands(cylc_version, name, log_file): f"{PROFILE} ; " f"export CYLC_VERSION={cylc_version} ; " f"cylc stop '{name}' >/dev/null 2>&1 ; sleep 10 ; " - f"{CYLC_DIFFS[major_cylc_version(cylc_version)]['clean']} -y -q {name} " + f"cylc clean --timeout=7200 -y -q {name} " f">> {log_file} 2>&1\n" ) @@ -252,38 +200,22 @@ def generate_clean_cron(suite_name, suite, log_file, cylc_version): return clean_cron -def generate_rose_stem_command(suite, wc_path, cylc_version, name): +def generate_cylc_command(suite, wc_path, cylc_version, name): """ Return a string with the rose-stem command Ignores any additional source arguments """ - return ( + command = ( f"export CYLC_VERSION={cylc_version} ; " - + f"rose stem --group={suite['groups']} " - + f"{CYLC_DIFFS[major_cylc_version(cylc_version)]['name']}{name} " - + f"--source={wc_path} " + f"cylc vip -z g={suite['groups']} " + f"-n {name} " + f"-S USE_MIRRORS=true " ) - - -def populate_heads_sources(suite): - """ - Append all required dependency sources when using heads testing - """ - - heads = "" - - if ( - "revisions" not in suite - or suite["revisions"] != "heads" - or suite["repo"] == "lfric_apps" - or suite["repo"] not in DEPENDENCIES - ): - return heads - - for item in sorted(DEPENDENCIES[suite["repo"]]): - heads += f"--source=fcm:{item}.xm_tr@HEAD " - return heads + if "revisions" in suite and suite["revisions"] == "heads": + command += "-S USE_HEADS=true " + command += f"{os.path.join(wc_path, "rose-stem")} " + return command def populate_cl_variables(suite): @@ -317,19 +249,13 @@ def generate_main_job(name, suite, log_file, wc_path, cylc_version): wc_path = wc_path + "_heads" # Begin rose-stem command - job_command += generate_rose_stem_command(suite, wc_path, cylc_version, name) - - # Add Heads Sources if required - job_command += populate_heads_sources(suite) + job_command += generate_cylc_command(suite, wc_path, cylc_version, name) # Add any -S vars defined job_command += populate_cl_variables(suite) job_command += f">> {log_file} 2>&1" - if major_cylc_version(cylc_version) == "8": - job_command += f" ; cylc play {name} >> {log_file} 2>&1" - # If this is a cylc-8-next job, check that the 8-next symlink in metomi points # elsewhere than the cylc-8 symlink if cylc_version == "8-next": @@ -351,12 +277,12 @@ def generate_cron_job(suite_name, suite, log_file): rose-stem task and for the suite-clean task """ - cylc_version = suite.get("cylc_version", DEFAULT_CYLC_VERSION) + cylc_version = suite.get("cylc_version", "8") cylc_version = str(cylc_version) date_str = f"_$({DATE_BASE})" name = suite_name + date_str - wc_path = os.path.join(WC_DIR, "wc_" + suite["repo"]) + wc_path = os.path.join(CLONE_DIR, "wc_" + suite["repo"]) header = generate_header(suite_name, suite) cron_job = generate_main_job(name, suite, log_file, wc_path, cylc_version) @@ -381,7 +307,7 @@ def parse_cl_args(): parser.add_argument( "-f", "--cron_file", - default="~/Crontabs/auto-gen_testing.cron", + default="~/Crontabs/auto-gen_testing_git.cron", help="The file the cronjobs will be written to." "Installation assumes this ends with '.cron'", ) @@ -422,7 +348,8 @@ def parse_cl_args(): "'generate_test-suite_cron.py' file.\n# Use that script and associated " "config file to modify these cron jobs.\n\n" ) - main_crontab += fetch_working_copy_cron() + for repo in DEPENDENCIES: + main_crontab += create_git_clone_cron(repo) last_repo = None for suite_name in sorted(suites.keys()): diff --git a/nightly_testing/generate_test_suite_cron_fcm.py b/nightly_testing/generate_test_suite_cron_fcm.py new file mode 100755 index 0000000..5c940c9 --- /dev/null +++ b/nightly_testing/generate_test_suite_cron_fcm.py @@ -0,0 +1,455 @@ +#!/usr/bin/env python3 +# *****************************COPYRIGHT******************************* +# (C) Crown copyright Met Office. All rights reserved. +# For further details please refer to the file COPYRIGHT.txt +# which you should have received as part of this distribution. +# *****************************COPYRIGHT******************************* + +""" +Script to generate a cron file, from which nightly testing is run. +Cron jobs are based on a provided yaml config file (-c command line option) and +the resulting cron file is installed if the --install option is added. +See https://metoffice.github.io/simulation-systems/Reviewers/nightlytesting.html +Used for nightly testing for UM, Jules, UKCA, LFRic_Apps +Compatible for both Cylc7 + Cylc8 + +YAML Config Options: +Required: +* repo: the repo being run +* time_launch: HH:MM, the time to start the run suite cronjob +* time_clean: HH:MM, the time to start the clean suite cronjob - for a given + task this will be delayed by the amount of time given by the + period value +* period: 'weekly' runs on Monday, cleans on Sunday. 'nightly' runs Tue-Fri, + cleans Wed-Sat. 'nightly_all' runs Mon-Fri, cleans Tue-Sat +* groups: the groups to run +Optional: +* revisions: heads to use the HoT for sub-repos, set to use the set revisions +* vars: strings that follow the -S command on the command line +* monitoring: Boolean, whether to run the monitoring script on this suite +* cylc_version: Can be any string beginning 7 or 8 that is a valid cylc install + at the site, such that `export CYLC_VERSION=` + works. +""" + +import argparse +import os +import re +import subprocess +import sys + +import yaml + +DEFAULT_CYLC_VERSION = "8" +DEPENDENCIES = { + "lfric_apps": [ + "casim", + "jules", + "lfric", + "shumlib", + "socrates", + "ukca", + "um", + ], + "um": ["casim", "jules", "mule", "shumlib", "socrates", "ukca"], + "lfric": [], + "jules": [], + "ukca": [], +} +CYLC_DIFFS = { + "7": { + "name": "--name=", + "clean": "rose suite-clean", + }, + "8": { + "name": "--workflow-name=", + "clean": "cylc clean --timeout=7200", + }, +} + +WC_DIR = os.path.join(os.environ["TMPDIR"], os.environ["USER"]) +UMDIR = os.environ["UMDIR"] +PROFILE = ". /etc/profile" +DATE_BASE = "date +\\%Y-\\%m-\\%d" +MONITORING_TIME = "00 06" + + +def run_command(command): + """ + Run a subprocess command and return the result object + """ + return subprocess.run( + command, shell=True, capture_output=True, text=True, timeout=5 + ) + + +def major_cylc_version(cylc_version): + """ + Return the major version of cylc being requested by cylc_version + Expected to be 7 or 8 + """ + return re.split("[._-]", cylc_version)[0] + + +def join_checkout_commands(repos, dir_wc): + """ + Join commands that checkout new repos + """ + + command = "" + for repo in repos: + wc_path = os.path.join(dir_wc, "wc_" + repo) + command += f"fcm co -q --force fcm:{repo}.xm_tr@HEAD {wc_path} ; " + return command + + +def fetch_working_copy_cron(): + """ + Cleanup and then re-checkout working copies for each of the repos used + Create an lfric_apps heads working copy + Runs at 23:30, before all other tasks + """ + + command = "# Checkout Working Copies - every day at 23:30 #" + l = len(command) + command = f"{l*'#'}\n{command}\n{l*'#'}\n30 23 * * * {PROFILE} ; " + command += f"rm -rf {os.path.join(WC_DIR, 'wc_*')} ; " + command += join_checkout_commands(DEPENDENCIES.keys(), WC_DIR) + command += lfric_heads_sed(os.path.join(WC_DIR, "wc_lfric_apps")) + + return command + "\n\n\n" + + +def lfric_heads_sed(wc_path): + """ + Add 2 sed commands to setup dependencies.sh for heads testing + One command replaces all revisions with HEAD + The other removes all sources - in the event a branch/working copy is left in at + commit, this means that the Heads testing should still run. The set-revisions + testing would show the wrong source. + As this edits the working copy it copies the original copy with _heads added + and returns this new wc path + """ + + wc_path_new = wc_path + "_heads" + dep_path = os.path.join(wc_path_new, "dependencies.sh") + + rstr = f"cp -rf {wc_path} {wc_path_new} ; " + rstr += f'sed -i -e "s/^\\(export .*_rev=\\).*/\\1HEAD/" {dep_path} ; ' + rstr += f'sed -i -e "s/^\\(export .*_sources=\\).*/\\1/" {dep_path} ; ' + return rstr + + +def generate_cron_timing_str(suite, mode): + """ + Return a string with the cron timing info included but no commands + """ + + if mode == "monitoring": + cron = f"{MONITORING_TIME}" + elif mode == "main": + cron = suite["cron_launch"] + elif mode == "clean": + cron = suite["cron_clean"] + else: + sys.exit("Unrecognised mode for cron timing string") + cron += " * * " + + if suite["period"] == "weekly": + if mode == "main" or mode == "monitoring": + cron += "1 " + else: + cron += "7 " + elif suite["period"] == "nightly_all": + if mode == "main" or mode == "monitoring": + cron += "1-5 " + else: + cron += "2-6 " + else: + if mode == "main" or mode == "monitoring": + cron += "2-5 " + else: + cron += "3-6 " + return cron + + +def generate_header(name, suite): + """ + Generate a comment describing the suite + """ + + header = f"# {name} #" + hashes = "".join("#" for _ in header) + header = f"{hashes}\n{header}\n{hashes}\n" + header += f"# Launch at {suite['time_launch']} on " + if suite["period"] == "weekly": + header += "Mon\n" + elif suite["period"] == "nightly_all": + header += "Mon-Fri\n" + else: + header += "Tue-Fri\n" + header += f"# Clean at {suite['time_clean']} on " + if suite["period"] == "weekly": + header += "Sun\n" + elif suite["period"] == "nightly_all": + header += "Tue-Sat\n" + else: + header += "Wed-Sat\n" + return header + + +def generate_monitoring(name, suite, log_file): + """ + Generate the monitoring command cron job + Default to off if not specified in config + """ + + # Return empty string if not required - default to this state + if "monitoring" not in suite or not suite["monitoring"]: + return "" + + script = os.path.join(UMDIR, "bin", "monitoring.py") + cylc_dir = os.path.expanduser(os.path.join("~", "cylc-run", name)) + + monitoring = generate_cron_timing_str(suite, "monitoring") + + monitoring += ( + f"{PROFILE} ; module load scitools/default-current ; " + f"{script} {cylc_dir} >> {log_file} 2>&1" + ) + + return monitoring + "\n" + + +def generate_clean_commands(cylc_version, name, log_file): + """ + Generate the commands used to clean the suite + """ + return ( + f"{PROFILE} ; " + f"export CYLC_VERSION={cylc_version} ; " + f"cylc stop '{name}' >/dev/null 2>&1 ; sleep 10 ; " + f"{CYLC_DIFFS[major_cylc_version(cylc_version)]['clean']} -y -q {name} " + f">> {log_file} 2>&1\n" + ) + + +def generate_clean_cron(suite_name, suite, log_file, cylc_version): + """ + Return a string of the cronjob for cleaning the suite + """ + + clean_cron = generate_cron_timing_str(suite, "clean") + if suite["period"] == "weekly": + date_str = f'_$({DATE_BASE} -d "6 days ago")' + else: + date_str = f'_$({DATE_BASE} -d "1 day ago")' + + name = suite_name + date_str + + clean_cron += generate_clean_commands(cylc_version, name, log_file) + + return clean_cron + + +def generate_rose_stem_command(suite, wc_path, cylc_version, name): + """ + Return a string with the rose-stem command + Ignores any additional source arguments + """ + + return ( + f"export CYLC_VERSION={cylc_version} ; " + + f"rose stem --group={suite['groups']} " + + f"{CYLC_DIFFS[major_cylc_version(cylc_version)]['name']}{name} " + + f"--source={wc_path} " + ) + + +def populate_heads_sources(suite): + """ + Append all required dependency sources when using heads testing + """ + + heads = "" + + if ( + "revisions" not in suite + or suite["revisions"] != "heads" + or suite["repo"] == "lfric_apps" + or suite["repo"] not in DEPENDENCIES + ): + return heads + + for item in sorted(DEPENDENCIES[suite["repo"]]): + heads += f"--source=fcm:{item}.xm_tr@HEAD " + return heads + + +def populate_cl_variables(suite): + """ + Combine variables with the -S command line argument + """ + + cl_vars = "" + + if "vars" not in suite: + return cl_vars + + for item in suite["vars"]: + cl_vars += f"-S {item} " + + return cl_vars + + +def generate_main_job(name, suite, log_file, wc_path, cylc_version): + """ + Generate the main cron_job commands + """ + + # Set up the timing for this job + cron_job = generate_cron_timing_str(suite, "main") + + job_command = f"{PROFILE} ; " + + # LFRic Apps heads uses a different working copy + if suite["repo"] == "lfric_apps" and suite["revisions"] == "heads": + wc_path = wc_path + "_heads" + + # Begin rose-stem command + job_command += generate_rose_stem_command(suite, wc_path, cylc_version, name) + + # Add Heads Sources if required + job_command += populate_heads_sources(suite) + + # Add any -S vars defined + job_command += populate_cl_variables(suite) + + job_command += f">> {log_file} 2>&1" + + if major_cylc_version(cylc_version) == "8": + job_command += f" ; cylc play {name} >> {log_file} 2>&1" + + # If this is a cylc-8-next job, check that the 8-next symlink in metomi points + # elsewhere than the cylc-8 symlink + if cylc_version == "8-next": + next_link = os.path.join(CYLC_INSTALL, "cylc-8-next") + def_link = os.path.join(CYLC_INSTALL, "cylc-8") + cron_job += ( + f'[ "$(readlink -- {next_link})" != "$(readlink -- {def_link})" ] ' + f"&& ({job_command})" + ) + else: + cron_job += job_command + + return cron_job + "\n" + + +def generate_cron_job(suite_name, suite, log_file): + """ + Using the suite settings from the config file, define a cronjob for the + rose-stem task and for the suite-clean task + """ + + cylc_version = suite.get("cylc_version", DEFAULT_CYLC_VERSION) + cylc_version = str(cylc_version) + + date_str = f"_$({DATE_BASE})" + name = suite_name + date_str + wc_path = os.path.join(WC_DIR, "wc_" + suite["repo"]) + + header = generate_header(suite_name, suite) + cron_job = generate_main_job(name, suite, log_file, wc_path, cylc_version) + monitoring = generate_monitoring(name, suite, log_file) + clean_cron = generate_clean_cron(suite_name, suite, log_file, cylc_version) + + return header + cron_job + monitoring + clean_cron + + +def parse_cl_args(): + """ + Parse command line arguments + """ + + parser = argparse.ArgumentParser("Generate Cronjobs for nightly testing") + parser.add_argument( + "-c", + "--config", + required=True, + help="Path to a yaml config file with suites defined.", + ) + parser.add_argument( + "-f", + "--cron_file", + default="~/Crontabs/auto-gen_testing.cron", + help="The file the cronjobs will be written to." + "Installation assumes this ends with '.cron'", + ) + parser.add_argument( + "-l", + "--cron_log", + default="~/cron.log", + help="The file any stdout or stderr will be piped to.", + ) + parser.add_argument( + "-p", + "--cylc_path", + default="~metomi/apps", + help="The location of the cylc installation required for testing `next-cylc`" + "configs.", + ) + parser.add_argument( + "--install", + action="store_true", + help="If True, will install the generated crontab.", + ) + args = parser.parse_args() + args.cron_file = os.path.expanduser(args.cron_file) + args.cron_log = os.path.expanduser(args.cron_log) + global CYLC_INSTALL + CYLC_INSTALL = args.cylc_path + return args + + +if __name__ == "__main__": + args = parse_cl_args() + + with open(args.config) as stream: + suites = yaml.safe_load(stream) + + main_crontab = ( + "# WARNING: This file is automatically generated by the " + "'generate_test-suite_cron.py' file.\n# Use that script and associated " + "config file to modify these cron jobs.\n\n" + ) + main_crontab += fetch_working_copy_cron() + + last_repo = None + for suite_name in sorted(suites.keys()): + if suite_name == "base": + continue + repo = suites[suite_name]["repo"] + tlaunch = suites[suite_name]["time_launch"].split(":") + tclean = suites[suite_name]["time_clean"].split(":") + suites[suite_name]["cron_launch"] = f"{tlaunch[1]} {tlaunch[0]}" + suites[suite_name]["cron_clean"] = f"{tclean[1]} {tclean[0]}" + if repo != last_repo: + main_crontab += 80 * "#" + "\n" + main_crontab += f"# {repo.upper()} SUITES\n" + main_crontab += 80 * "#" + 2 * "\n" + last_repo = repo + main_crontab += generate_cron_job(suite_name, suites[suite_name], args.cron_log) + main_crontab += 3 * "\n" + + with open(args.cron_file, "w") as outfile: + outfile.write(main_crontab) + + # Install any file with .cron extension in the specified dir + cron_path = args.cron_file.strip(os.path.basename(args.cron_file)) + all_file = os.path.join(cron_path, "all_cron_jobs.cron") + if args.install: + command = f"cat {os.path.join(cron_path, '*.cron')} | crontab -" + result = run_command(command) + if result.returncode: + print("Failed to install crontab. Error:") + sys.exit(result.stderr)