diff --git a/krun.py b/krun.py index ce117e1..2832d88 100755 --- a/krun.py +++ b/krun.py @@ -11,7 +11,9 @@ import locale import krun.util as util +from krun.config import Config from krun.platform import detect_platform +from krun.results import Results from krun.scheduler import ExecutionScheduler from krun import ABS_TIME_FORMAT from krun.mail import Mailer @@ -40,21 +42,21 @@ def usage(parser): def sanity_checks(config, platform): vms_that_will_run = [] # check all necessary benchmark files exist - for bench, bench_param in config["BENCHMARKS"].items(): - for vm_name, vm_info in config["VMS"].items(): + for bench, bench_param in config.BENCHMARKS.items(): + for vm_name, vm_info in config.VMS.items(): for variant in vm_info["variants"]: - entry_point = config["VARIANTS"][variant] + entry_point = config.VARIANTS[variant] key = "%s:%s:%s" % (bench, vm_name, variant) debug("Running sanity check for experiment %s" % key) - if util.should_skip(config, key): + if config.should_skip(key): continue # won't execute, so no check needed vm_info["vm_def"].check_benchmark_files(bench, entry_point) vms_that_will_run.append(vm_name) # per-VM sanity checks - for vm_name, vm_info in config["VMS"].items(): + for vm_name, vm_info in config.VMS.items(): if vm_name not in vms_that_will_run: # User's SKIP config directive may mean a defined VM never runs. # This may be deliberate, e.g. the user does not yet have it built. @@ -101,109 +103,90 @@ def sanity_check_user_change(platform): def create_arg_parser(): """Create a parser to process command-line options. """ - parser = argparse.ArgumentParser(description='Benchmark, running many fresh processes.') + parser = argparse.ArgumentParser(description="Benchmark, running many fresh processes.") # Upper-case '-I' so as to make it harder to use by accident. # Real users should never use -I. Only the OS init system. - parser.add_argument('--started-by-init', '-I', action='store_true', - default=False, dest='started_by_init', required=False, - help='Krun is being invoked by OS init system') - parser.add_argument('--resume', '-r', action='store_true', default=False, - dest='resume', required=False, + parser.add_argument("--started-by-init", "-I", action="store_true", + default=False, dest="started_by_init", required=False, + help="Krun is being invoked by OS init system") + parser.add_argument("--resume", "-r", action="store_true", default=False, + dest="resume", required=False, help=("Resume benchmarking if interrupted " + "and append to an existing results file")) - parser.add_argument('--reboot', '-b', action='store_true', default=False, - dest='reboot', required=False, - help='Reboot before each benchmark is executed') - parser.add_argument('--dryrun', '-d', action='store_true', default=False, - dest='dry_run', required=False, + parser.add_argument("--reboot", "-b", action="store_true", default=False, + dest="reboot", required=False, + help="Reboot before each benchmark is executed") + parser.add_argument("--dryrun", "-d", action="store_true", default=False, + dest="dry_run", required=False, help=("Build and run a benchmarking schedule " + "But don't execute the benchmarks. " + "Useful for verifying configuration files")) - parser.add_argument('--debug', '-g', action="store", default='INFO', - dest='debug_level', required=False, - help=('Debug level used by logger. Must be one of: ' + - 'DEBUG, INFO, WARN, DEBUG, CRITICAL, ERROR')) - parser.add_argument('--dump-audit', action="store_true", - dest='dump_audit', required=False, - help=('Print the audit section of a Krun ' + - 'results file to STDOUT')) - parser.add_argument('--dump-config', action="store_true", - dest='dump_config', required=False, - help=('Print the config section of a Krun ' + - 'results file to STDOUT')) - parser.add_argument('--dump-reboots', action="store_true", - dest='dump_reboots', required=False, - help=('Print the reboots section of a Krun ' + - 'results file to STDOUT')) - parser.add_argument('--dump-etas', action="store_true", - dest='dump_etas', required=False, - help=('Print the eta_estimates section of a Krun ' + - 'results file to STDOUT')) - parser.add_argument('--dump-temps', action="store_true", - dest='dump_temps', required=False, - help=('Print the starting_temperatures section of ' + - 'a Krun results file to STDOUT')) - parser.add_argument('--dump-data', action="store_true", - dest='dump_data', required=False, - help=('Print the data section of ' + - 'a Krun results file to STDOUT')) - parser.add_argument('--develop', action="store_true", - dest='develop', required=False, - help=('Enable developer mode')) - filename_help = ('Krun configuration or results file. FILENAME should' + - ' be a configuration file when running benchmarks ' + - '(e.g. experiment.krun) and a results file ' + - '(e.g. experiment_results.json.bz2) when calling ' + - 'krun with --dump-config, --dump_audit or ' + - '--dump-reboots') - parser.add_argument('filename', action="store", # Required by default. - metavar='FILENAME', + parser.add_argument("--debug", "-g", action="store", default='INFO', + dest="debug_level", required=False, + help=("Debug level used by logger. Must be one of: " + + "DEBUG, INFO, WARN, DEBUG, CRITICAL, ERROR")) + parser.add_argument("--dump-audit", action="store_const", + dest="dump", const="audit", required=False, + help=("Print the audit section of a Krun " + + "results file to STDOUT")) + parser.add_argument("--dump-config", action="store_const", + dest="dump", const="config", required=False, + help=("Print the config section of a Krun " + + "results file to STDOUT")) + parser.add_argument("--dump-reboots", action="store_const", + dest="dump", const="reboots", required=False, + help=("Print the reboots section of a Krun " + + "results file to STDOUT")) + parser.add_argument("--dump-etas", action="store_const", + dest="dump", const="eta_estimates", required=False, + help=("Print the eta_estimates section of a Krun " + + "results file to STDOUT")) + parser.add_argument("--dump-temps", action="store_const", + dest="dump", const="starting_temperatures", + required=False, + help=("Print the starting_temperatures section of " + + "a Krun results file to STDOUT")) + parser.add_argument("--dump-data", action="store_const", + dest="dump", const="data", required=False, + help=("Print the data section of " + + "a Krun results file to STDOUT")) + parser.add_argument("--develop", action="store_true", + dest="develop", required=False, + help=("Enable developer mode")) + filename_help = ("Krun configuration or results file. FILENAME should" + + " be a configuration file when running benchmarks " + + "(e.g. experiment.krun) and a results file " + + "(e.g. experiment_results.json.bz2) when calling " + + "krun with --dump-config, --dump_audit, " + + "--dump-reboots, --dump-etas, --dump-temps, or" + "--dump-data") + parser.add_argument("filename", action="store", # Required by default. + metavar="FILENAME", help=(filename_help)) return parser -def dump_section(args): - results = util.read_results(args.filename) - if args.dump_config: - text = results['config'] - elif args.dump_audit: - text = util.dump_audit(results['audit']) - elif args.dump_reboots: - text = str(results['reboots']) - elif args.dump_etas: - text = json.dumps(results['eta_estimates'], - sort_keys=True, indent=2) - elif args.dump_temps: - text = json.dumps(results['starting_temperatures'], - sort_keys=True, indent=2) - elif args.dump_data: - text = json.dumps(results['data'], - sort_keys=True, indent=2) - else: - assert False # unreachable - - # String data read in from JSON are unicode objects. This matters for us - # as some data in the audit includes unicode characters. If it does, - # a simple print no longer suffices if the system locale is (e.g.) ASCII. - # In this case print will raise. - # - # The correct thing to do is to encode() the unicode to the system locale. - print(text.encode(locale.getpreferredencoding())) - def main(parser): args = parser.parse_args() - if (args.dump_config or - args.dump_audit or - args.dump_reboots or - args.dump_etas or - args.dump_temps or - args.dump_data): + if args.dump is not None: if not args.filename.endswith(".json.bz2"): usage(parser) else: - dump_section(args) + results = Results(None, None, results_file=args.filename) + if args.dump == "config" or "audit": + text = unicode(results.__getattribute__(args.dump)) + else: + text = json.dumps(results.__getattribute__(args.dump), + sort_keys=True, indent=2) + # String data read in from JSON are unicode objects. This matters + # for us as some data in the audit includes unicode characters. + # If it does, a simple print no longer suffices if the system + # locale is (e.g.) ASCII. In this case print will raise an + # exception. The correct thing to do is to encode() the unicode to + # the system locale. + print(text.encode(locale.getpreferredencoding())) sys.exit(0) if not args.filename.endswith(".krun"): @@ -215,8 +198,8 @@ def main(parser): except OSError: util.fatal('Krun configuration file %s does not exist.' % args.filename) - config = util.read_config(args.filename) - out_file = util.output_name(args.filename) + config = Config(args.filename) + out_file = config.results_filename() out_file_exists = os.path.exists(out_file) if out_file_exists and not os.path.isfile(out_file): @@ -240,12 +223,11 @@ def main(parser): if args.develop: warn("Developer mode enabled. Results will not be reliable.") - mail_recipients = config.get("MAIL_TO", []) + mail_recipients = config.MAIL_TO if type(mail_recipients) is not list: util.fatal("MAIL_TO config should be a list") - max_mails = config.get("MAX_MAILS", 5) - mailer = Mailer(mail_recipients, max_mails=max_mails) + mailer = Mailer(mail_recipients, max_mails=config.MAX_MAILS) # Initialise platform instance and assign to VM defs. # This needs to be done early, so VM sanity checks can run. @@ -260,8 +242,6 @@ def main(parser): platform.developer_mode = True platform.collect_audit() - for vm_name, vm_info in config["VMS"].items(): - vm_info["vm_def"].set_platform(platform) # If the user has asked for resume-mode, the current platform must # be an identical machine to the current one. @@ -273,12 +253,13 @@ def main(parser): if args.resume: # output file must exist, due to check above assert(out_file_exists) - current = util.read_results(out_file) - if not util.audits_same_platform(platform.audit, current["audit"]): + current = Results(config, platform, results_file=out_file) + from krun.audit import Audit + if not Audit(platform.audit) == current.audit: util.fatal(error_msg) debug("Using pre-recorded initial temperature readings") - platform.set_starting_temperatures(current["starting_temperatures"]) + platform.set_starting_temperatures(current.starting_temperatures) else: # Touch the config file to update its mtime. This is required # by resume-mode which uses the mtime to determine the name of @@ -290,21 +271,19 @@ def main(parser): debug("Taking fresh initial temperature readings") platform.set_starting_temperatures() - log_filename = attach_log_file(args.filename, args.resume) + attach_log_file(config, args.resume) sanity_checks(config, platform) # Build job queue -- each job is an execution - sched = ExecutionScheduler(args.filename, - log_filename, - out_file, + sched = ExecutionScheduler(config, mailer, platform, resume=args.resume, reboot=args.reboot, dry_run=args.dry_run, started_by_init=args.started_by_init) - sched.build_schedule(config, current) + sched.build_schedule() # does the benchmarking sched.run() @@ -331,13 +310,13 @@ def setup_logging(parser): logging.root.handlers = [stream] -def attach_log_file(config_filename, resume): - log_filename = util.log_name(config_filename, resume) +def attach_log_file(config, resume): + log_filename = config.log_filename(resume) mode = 'a' if resume else 'w' fh = logging.FileHandler(log_filename, mode=mode) fh.setFormatter(PLAIN_FORMATTER) logging.root.addHandler(fh) - return os.path.abspath(log_filename) + return if __name__ == "__main__": diff --git a/krun/__init__.py b/krun/__init__.py index 7b1a20d..a1f33bc 100644 --- a/krun/__init__.py +++ b/krun/__init__.py @@ -4,6 +4,12 @@ UNKNOWN_ABS_TIME = "????-??-?? ??:??:??" class EntryPoint(object): + def __init__(self, target, subdir=None): self.target = target self.subdir = subdir + + def __eq__(self, other): + return (isinstance(other, self.__class__) and + (self.target == other.target) and + (self.subdir == other.subdir)) diff --git a/krun/audit.py b/krun/audit.py new file mode 100644 index 0000000..cfc1463 --- /dev/null +++ b/krun/audit.py @@ -0,0 +1,50 @@ +from collections import OrderedDict +from logging import debug + + +class Audit(object): + + def __init__(self, audit_dict): + assert isinstance(audit_dict, dict) + self._audit = audit_dict + + def __contains__(self, key): + return key in self._audit + + def __getitem__(self, key): + return self._audit[key] + + def __setitem__(self, key, value): + self._audit[key] = value + + def __unicode__(self): + s = "" + # important that the sections are sorted, for diffing + for key, text in OrderedDict(sorted(self._audit.iteritems())).iteritems(): + s += "Audit Section: %s" % key + "\n" + s += "#" * 78 + "\n\n" + s += unicode(text) + "\n\n" + return s + + def __len__(self): + return len(self._audit) + + @property + def audit(self): + return self._audit + + @audit.setter + def audit(self, audit_dict): + self._audit = audit_dict + + def __eq__(self, other): + if ((not isinstance(other, self.__class__)) or + (not len(self) == len(other))): + return False + for key in self._audit: + if key == "packages" and (key in self) and (key in other): + continue + if ((key not in other._audit) or (other[key] != self[key])): + debug("%s keys differed in audit" % key) + return False + return True diff --git a/krun/config.py b/krun/config.py new file mode 100644 index 0000000..08ac238 --- /dev/null +++ b/krun/config.py @@ -0,0 +1,94 @@ +from logging import error + +import os.path +import time + +from krun import LOGFILE_FILENAME_TIME_FORMAT + + +class Config(object): + """All configuration for a Krun benchmark. + Includes CLI args as well as configuration from .krun files. + """ + + def __init__(self, config_file=None): + self.MAIL_TO = list() + self.MAX_MAILS = 5 + self.VMS = dict() + self.VARIANTS = dict() + self.BENCHMARKS = dict() + self.SKIP = list() + self.N_EXECUTIONS = 1 + self.filename = config_file + if config_file is not None: + self.read_from_file(config_file) + + def read_from_string(self, config_str): + config_dict = {} + try: + exec(config_str, config_dict) + except Exception as e: + error("error importing config file:\n%s" % str(e)) + raise + self.__dict__.update(config_dict) + self.filename = "" + self.text = config_str + + def read_from_file(self, config_file): + assert config_file.endswith(".krun") + config_dict = {} + try: + execfile(config_file, config_dict) + except Exception as e: + error("error importing config file:\n%s" % str(e)) + raise + self.__dict__.update(config_dict) + self.filename = config_file + with open(config_file, "r") as fp: + self.text = fp.read() + + def log_filename(self, resume=False): + assert self.filename.endswith(".krun") + if resume: + config_mtime = time.gmtime(os.path.getmtime(self.filename)) + tstamp = time.strftime(LOGFILE_FILENAME_TIME_FORMAT, config_mtime) + else: + tstamp = time.strftime(LOGFILE_FILENAME_TIME_FORMAT) + return self.filename[:-5] + "_%s.log" % tstamp + + def results_filename(self): # FIXME: was called output_name in util + """Makes a result file name based upon the config file name.""" + assert self.filename.endswith(".krun") + return self.filename[:-5] + "_results.json.bz2" + + def should_skip(self, this_key): + for skip_key in self.SKIP: + skip_elems = skip_key.split(":") + this_elems = this_key.split(":") + + # Should be triples of: bench * vm * variant + assert len(skip_elems) == 3 and len(this_elems) == 3 + + for i in range(3): + if skip_elems[i] == "*": + this_elems[i] = "*" + + if skip_elems == this_elems: + return True # skip + + return False + + def __str__(self): + return self.text + + def __eq__(self, other): + # Equality should ignore filename. + return (isinstance(other, self.__class__) and + (self.text == other.text) and + (self.MAIL_TO == other.MAIL_TO) and + (self.MAX_MAILS == other.MAX_MAILS) and + (self.VMS == other.VMS) and + (self.VARIANTS == other.VARIANTS) and + (self.BENCHMARKS == other.BENCHMARKS) and + (self.SKIP == other.SKIP) and + (self.N_EXECUTIONS == other.N_EXECUTIONS)) diff --git a/krun/results.py b/krun/results.py new file mode 100644 index 0000000..162807e --- /dev/null +++ b/krun/results.py @@ -0,0 +1,106 @@ +from krun.audit import Audit +from krun.config import Config + +import bz2 # decent enough compression with Python 2.7 compatibility. +import json + + +class Results(object): + """Results of a Krun benchmarking session. + Can be serialised to disk. + """ + + def __init__(self, config, platform, results_file=None): + self.config = config + + # Maps key to results: + # "bmark:vm:variant" -> [[e0i0, e0i1, ...], [e1i0, e1i1, ...], ...] + self.data = dict() + self.reboots = 0 + + # Record how long execs are taking so we can give the user a rough ETA. + # Maps "bmark:vm:variant" -> [t_0, t_1, ...] + self.eta_estimates = dict() + + # error_flag is flipped when a (non-fatal) error or warning occurs. + # When Krun finishes and this flag is true, a message is printed, + # thus prompting the user to investigate. + self.error_flag = False + + # Fill in attributes from the config, platform and prior results. + if self.config is not None: + self.filename = self.config.results_filename() + self.init_from_config() + if platform is not None: + self.starting_temperatures = platform.starting_temperatures + self._audit = Audit(platform.audit) + else: + self.starting_temperatures = list() + self.audit = dict() + + # Import data from a Results object serialised on disk. + if results_file is not None: + self.read_from_file(results_file) + + @property + def audit(self): + return self._audit + + @audit.setter + def audit(self, audit_dict): + self._audit = Audit(audit_dict) + + def init_from_config(self): + """Scaffold dictionaries based on a given configuration. + """ + # Initialise dictionaries based on config information. + for vm_name, vm_info in self.config.VMS.items(): + for bmark, _ in self.config.BENCHMARKS.items(): + for variant in vm_info["variants"]: + key = ":".join((bmark, vm_name, variant)) + self.data[key] = [] + self.eta_estimates[key] = [] + + def read_from_file(self, results_file): + """Initialise object from serialised file on disk. + """ + with bz2.BZ2File(results_file, "rb") as f: + results = json.loads(f.read()) + self.__dict__.update(results) + # Ensure that self.audir and self.config have correct types. + self.config = Config(None) + self.config.read_from_string(results["config"]) + self.audit = results["audit"] + + def write_to_file(self): + """Serialise object on disk. + """ + to_write = { + "config": self.config.text, + "data": self.data, + "audit": self.audit.audit, + "reboots": self.reboots, + "starting_temperatures": self.starting_temperatures, + "eta_estimates": self.eta_estimates, + "error_flag": self.error_flag, + } + with bz2.BZ2File(self.filename, "w") as f: + f.write(json.dumps(to_write, + indent=1, sort_keys=True, encoding='utf-8')) + + def jobs_completed(self, key): + """Return number of executions for which we have data for a given + benchmark / vm / variant triplet. + """ + return len(self.data[key]) + + def __eq__(self, other): + if not isinstance(other, self.__class__): + return False + return (self.config == other.config and + self.data == other.data and + self.audit == other.audit and + self.reboots == other.reboots and + self.starting_temperatures == other.starting_temperatures and + self.eta_estimates == other.eta_estimates and + self.error_flag == other.error_flag) diff --git a/krun/scheduler.py b/krun/scheduler.py index 03d33e4..e93d66c 100644 --- a/krun/scheduler.py +++ b/krun/scheduler.py @@ -1,4 +1,5 @@ from krun.time_estimate import TimeEstimateFormatter +from krun.results import Results from krun import util from collections import deque @@ -33,13 +34,12 @@ class ScheduleEmpty(Exception): class ExecutionJob(object): """Represents a single executions level benchmark run""" - def __init__(self, sched, config, vm_name, vm_info, benchmark, variant, parameter): + def __init__(self, sched, vm_name, vm_info, benchmark, variant, parameter): self.sched = sched self.vm_name, self.vm_info = vm_name, vm_info self.benchmark = benchmark self.variant = variant self.parameter = parameter - self.config = config # Used in results JSON and ETA dict self.key = "%s:%s:%s" % (self.benchmark, self.vm_name, self.variant) @@ -58,8 +58,7 @@ def __str__(self): def __eq__(self, other): return (self.sched == other.sched and self.key == other.key and - self.parameter == other.parameter and - self.config == other.config) + self.parameter == other.parameter) def add_exec_time(self, exec_time): """Feed back a rough execution time for ETA usage""" @@ -68,7 +67,7 @@ def add_exec_time(self, exec_time): def run(self, mailer, dry_run=False): """Runs this job (execution)""" - entry_point = self.config["VARIANTS"][self.variant] + entry_point = self.sched.config.VARIANTS[self.variant] vm_def = self.vm_info["vm_def"] vm_def.dry_run = dry_run @@ -83,7 +82,7 @@ def run(self, mailer, dry_run=False): tfmt.delta_str)) # Set heap limit - heap_limit_kb = self.config["HEAP_LIMIT"] + heap_limit_kb = self.sched.config.HEAP_LIMIT heap_limit_b = heap_limit_kb * 1024 # resource module speaks in bytes heap_t = (heap_limit_b, heap_limit_b) resource.setrlimit(resource.RLIMIT_DATA, heap_t) @@ -108,13 +107,14 @@ def run(self, mailer, dry_run=False): class ExecutionScheduler(object): """Represents our entire benchmarking session""" - def __init__(self, config_file, log_filename, out_file, mailer, platform, + def __init__(self, config, mailer, platform, resume=False, reboot=False, dry_run=False, started_by_init=False): self.mailer = mailer + self.config = config self.work_deque = deque() - self.eta_avail = None + self.eta_avail = 0 self.jobs_done = 0 self.platform = platform self.resume = resume @@ -122,28 +122,17 @@ def __init__(self, config_file, log_filename, out_file, mailer, platform, self.dry_run = dry_run self.started_by_init = started_by_init - # Flipped when a (non-fatal) error or warning occurs. - # When krun finishes and this flag is true, a message is printed, - # thus prompting the user to investigate. - self.error_flag = False - - # Record how long processes are taking so we can make a - # rough ETA for the user. - # Maps "bmark:vm:variant" -> [t_0, t_1, ...] - self.eta_estimates = {} - - # Maps key to results: - # "bmark:vm:variant" -> [[e0i0, e0i1, ...], [e1i0, e1i1, ...], ...] - self.results = {} + if resume: + self.results = Results(self.config, platform, + results_file=self.config.results_filename()) + else: + self.results = Results(self.config, platform) - # Reboot mode. - self.nreboots = 0 - self.expected_reboots = 0 + # Give VM objects access to the platform. + for vm_name, vm_info in self.config.VMS.items(): + vm_info["vm_def"].set_platform(platform) - # file names - self.config_file = config_file - self.out_file = out_file - self.log_path = log_filename + self.log_path = self.config.log_filename(self.resume) def set_eta_avail(self): """call after adding job before eta should become available""" @@ -175,7 +164,7 @@ def __len__(self): return len(self.work_deque) def get_estimated_exec_duration(self, key): - previous_exec_times = self.eta_estimates.get(key) + previous_exec_times = self.results.eta_estimates.get(key) if previous_exec_times: return mean(previous_exec_times) else: @@ -194,17 +183,17 @@ def get_overall_time_estimate_formatter(self): return TimeEstimateFormatter(self.get_estimated_overall_duration()) def add_eta_info(self, key, exec_time): - self.eta_estimates[key].append(exec_time) + self.results.eta_estimates[key].append(exec_time) - def build_schedule(self, config, current_result_json): + def build_schedule(self): one_exec_scheduled = False eta_avail_job = None - for exec_n in xrange(config["N_EXECUTIONS"]): - for vm_name, vm_info in config["VMS"].items(): - for bmark, param in config["BENCHMARKS"].items(): + for exec_n in xrange(self.config.N_EXECUTIONS): + for vm_name, vm_info in self.config.VMS.items(): + for bmark, param in self.config.BENCHMARKS.items(): for variant in vm_info["variants"]: - job = ExecutionJob(self, config, vm_name, vm_info, bmark, variant, param) - if not util.should_skip(config, job.key): + job = ExecutionJob(self, vm_name, vm_info, bmark, variant, param) + if not self.config.should_skip(job.key): if one_exec_scheduled and not eta_avail_job: # first job of second executions eta becomes known. eta_avail_job = job @@ -219,42 +208,40 @@ def build_schedule(self, config, current_result_json): # Resume mode: if previous results are available, remove the # jobs from the schedule which have already been executed, and # add the results to this object, ready to be saved to a Json file. - if self.resume and current_result_json is not None: - self._remove_previous_execs_from_schedule(current_result_json) - - # Load back (and sanity check) ETA estimates - self.eta_estimates = current_result_json["eta_estimates"] - for key, exec_data in current_result_json["data"].iteritems(): - got_len = len(current_result_json["eta_estimates"][key]) + if self.resume: + self._remove_previous_execs_from_schedule() + # Sanity check ETA estimates + for key, exec_data in self.results.data.iteritems(): + got_len = len(self.results.eta_estimates[key]) expect_len = len(exec_data) if expect_len != got_len: msg = "ETA estimates didn't tally with results: " - msg += "key=%s, expect_len=%d, got_len=%d" % \ + msg += "key=%s, expect_len=%d, got_len=%d " % \ (key, expect_len, got_len) + msg += "data[%s]=%s; " % (key, str(self.results.data[key])) + msg += "eta[%s]=%s" % \ + (key, str(self.results.eta_estimates[key])) util.log_and_mail(self.mailer, error, "Fatal Krun Error", msg, bypass_limiter=True, exit=True) - # Load back the error flag - self.error_flag = current_result_json["error_flag"] - - def _remove_previous_execs_from_schedule(self, current_result_json): - self.nreboots = current_result_json['reboots'] - for key in current_result_json['data']: - num_completed_jobs = len(current_result_json['data'][key]) - if num_completed_jobs > 0: - try: - debug("%s has already been run %d times." % - (key, num_completed_jobs)) - for _ in range(num_completed_jobs): - self.remove_job_by_key(key) - self.jobs_done += 1 - except JobMissingError as excn: - tup = (excn.key, self.config_file, self.out_file) - msg = ("Failed to resume benchmarking session\n." + - "The execution %s appears in results " + - "file: %s, but not in config file: %s." % tup) - util.fatal(msg) - self.results[key] = current_result_json['data'][key] + + def _remove_previous_execs_from_schedule(self): + for key in self.results.data: + num_completed_jobs = self.results.jobs_completed(key) + if num_completed_jobs > 0: + try: + debug("%s has already been run %d times." % + (key, num_completed_jobs)) + for _ in range(num_completed_jobs): + self.remove_job_by_key(key) + self.jobs_done += 1 + except JobMissingError as excn: + tup = (excn.key, self.config.filename, + self.config.results_filename()) + msg = ("Failed to resume benchmarking session\n." + + "The execution %s appears in results " + + "file: %s, but not in config file: %s." % tup) + util.fatal(msg) def run(self): """Benchmark execution starts here""" @@ -269,13 +256,6 @@ def run(self): self.log_path, bypass_limiter=True) - # scaffold dicts - for job in self.work_deque: - if not job.key in self.eta_estimates: - self.eta_estimates[job.key] = [] - if not job.key in self.results: - self.results[job.key] = [] - if self.reboot and not self.started_by_init: # Reboot before first benchmark (dumps results file). info("Reboot prior to first execution") @@ -329,19 +309,20 @@ def run(self): exec_result = util.format_raw_exec_results(raw_exec_result) if not exec_result and not self.dry_run: - self.error_flag = True + self.results.error_flag = True - self.results[job.key].append(exec_result) + self.results.data[job.key].append(exec_result) self.add_eta_info(job.key, exec_end_time - exec_start_time) # We dump the json after each experiment so we can monitor the # json file mid-run. It is overwritten each time. - info("Intermediate results dumped to %s" % self.out_file) - util.dump_results(self) + info("Intermediate results dumped to %s" % + self.config.results_filename()) + self.results.write_to_file() self.jobs_done += 1 if self.platform.check_dmesg_for_changes(): - self.error_flag = True + self.results.error_flag = True if self.reboot and len(self) > 0: info("Reboot in preparation for next execution") @@ -351,8 +332,8 @@ def run(self): self.platform.save_power() - info("Done: Results dumped to %s" % self.out_file) - if self.error_flag: + info("Done: Results dumped to %s" % self.config.results_filename()) + if self.results.error_flag: warn("Errors/warnings occurred -- read the log!") msg = "Completed in (roughly) %f seconds.\nLog file at: %s" % \ @@ -361,24 +342,25 @@ def run(self): bypass_limiter=True) def _reboot(self): - self.nreboots += 1 + self.results.reboots += 1 debug("About to execute reboot: %g, expecting %g in total." % - (self.nreboots, self.expected_reboots)) + (self.results.reboots, self.expected_reboots)) # Dump the results file. This may already have been done, but we # have changed self.nreboots, which needs to be written out. - util.dump_results(self) + self.results.write_to_file() - if self.nreboots > self.expected_reboots: + if self.results.reboots > self.expected_reboots: util.fatal(("HALTING now to prevent an infinite reboot loop: " + "INVARIANT num_reboots <= num_jobs violated. " + "Krun was about to execute reboot number: %g. " + "%g jobs have been completed, %g are left to go.") % - (self.nreboots, self.jobs_done, len(self))) + (self.results.reboots, self.jobs_done, len(self))) if self.dry_run: info("SIMULATED: reboot (restarting Krun in-place)") args = sys.argv if not self.started_by_init: args.extend(["--resume", "--started-by-init"]) + debug("Simulated reboot with args: " + " ".join(args)) os.execv(args[0], args) # replace myself assert False # unreachable else: diff --git a/krun/tests/skips.krun b/krun/tests/skips.krun new file mode 100644 index 0000000..87f250c --- /dev/null +++ b/krun/tests/skips.krun @@ -0,0 +1,55 @@ +import os +from krun.vm_defs import (PythonVMDef, JavaVMDef) +from krun import EntryPoint + +# Who to mail +MAIL_TO = [] + +# Maximum number of error emails to send per-run +#MAX_MAILS = 2 + +DIR = os.getcwd() +JKRUNTIME_DIR = os.path.join(DIR, "krun", "libkruntime", "") + +HEAP_LIMIT = 2097152 # K == 2Gb + +# Variant name -> EntryPoint +VARIANTS = { + "default-java": EntryPoint("KrunEntry", subdir="java"), + "default-python": EntryPoint("bench.py", subdir="python"), +} + +ITERATIONS_ALL_VMS = 1 # Small number for testing. + +VMS = { + 'Java': { + 'vm_def': JavaVMDef('/usr/bin/java'), + 'variants': ['default-java'], + 'n_iterations': ITERATIONS_ALL_VMS, + }, + 'CPython': { + 'vm_def': PythonVMDef('/usr/bin/python2'), + 'variants': ['default-python'], + 'n_iterations': ITERATIONS_ALL_VMS, + } +} + + +BENCHMARKS = { + 'dummy': 1000, + 'nbody': 1000, +} + +# list of "bench:vm:variant" +SKIP=[ + "*:PyPy:*", + "*:CPython:*", + "*:Hotspot:*", + "*:Graal:*", + "*:LuaJIT:*", + "*:HHVM:*", + "*:JRubyTruffle:*", + "*:V8:*", +] + +N_EXECUTIONS = 1 # Number of fresh processes. diff --git a/krun/tests/test_audit.py b/krun/tests/test_audit.py new file mode 100644 index 0000000..c7967c6 --- /dev/null +++ b/krun/tests/test_audit.py @@ -0,0 +1,49 @@ +from krun.audit import Audit + + +def test_eq(): + audit = Audit({"a":100, "b":200}) + empty = Audit(dict()) + assert audit == audit + assert empty == empty + assert not empty == audit + assert not list() == audit + audit["c"] = 300 + audit_ = Audit({"a":100, "b":200, "c":400}) + assert not audit == audit_ + + +def test_get_set_item(): + audit = Audit({"a":100, "b":200}) + empty = Audit(dict()) + assert audit["a"] == 100 + assert audit["b"] == 200 + empty["a"] = 100 + empty["b"] = 200 + assert audit == empty + + +def test_contains(): + audit = Audit({"a":100, "b":200}) + assert "a" in audit + assert not "c" in audit + + +def test_property(): + audit = Audit({"a":100, "b":200}) + assert audit.audit == {"a":100, "b":200} + empty = Audit(dict()) + empty.audit = {"a":100, "b":200} + assert empty == audit + + +def test_unicode(): + audit = Audit({"a":100, "b":200}) + spacer = "#" * 78 + expected = "Audit Section: a\n" + expected += spacer + u"\n\n" + expected += "100\n\n" + expected += "Audit Section: b\n" + expected += spacer + "\n\n" + expected += "200\n\n" + assert unicode(expected) == unicode(audit) diff --git a/krun/tests/test_config.py b/krun/tests/test_config.py new file mode 100644 index 0000000..0cda36e --- /dev/null +++ b/krun/tests/test_config.py @@ -0,0 +1,112 @@ +from krun import LOGFILE_FILENAME_TIME_FORMAT +from krun.config import Config + +import os +import pytest +import time + + +def touch(fname): + with open(fname, 'a'): + os.utime(fname, None) + + +def test_str(): + config = Config("krun/tests/example.krun") + assert config.text == str(config) + +def test_eq(): + example_config = Config("krun/tests/example.krun") + assert example_config == example_config + assert not example_config == None + assert not example_config == Config("krun/tests/quick.krun") + + +def test_log_filename(monkeypatch): + example_config = Config("krun/tests/example.krun") + touch("krun/tests/.krun") + empty_config = Config("krun/tests/.krun") + tstamp = time.strftime(LOGFILE_FILENAME_TIME_FORMAT) + assert example_config.log_filename(False) == "krun/tests/example" + "_" + tstamp + ".log" + assert empty_config.log_filename(False) == "krun/tests/_" + tstamp + ".log" + def mock_mtime(path): + return 1445964109.9363003 + monkeypatch.setattr(os.path, 'getmtime', mock_mtime) + tstamp = '20151027_164149' + assert example_config.log_filename(True) == "krun/tests/example" + "_" + tstamp + ".log" + assert empty_config.log_filename(True) == "krun/tests/_" + tstamp + ".log" + os.unlink("krun/tests/.krun") + + +def test_read_config_from_file(): + path = "krun/tests/example.krun" + config0 = Config(path) + config1 = Config(None) + config1.read_from_file(path) + assert config0 == config1 + + +def test_read_config_from_string(): + path = "krun/tests/example.krun" + config0 = Config(path) + config1 = Config(None) + with open(path) as fp: + config_string = fp.read() + config1.read_from_string(config_string) + assert config0 == config1 + + +def test_read_corrupt_config_from_string(): + path = "krun/tests/corrupt.krun" + config = Config(None) + with pytest.raises(Exception): + with open(path) as fp: + config_string = fp.read() + config.read_from_string(config_string) + + +def test_config_init(): + path = "examples/example.krun" + config = Config(path) + assert config is not None + assert config.BENCHMARKS == {"dummy": 1000, "nbody": 1000} + assert config.N_EXECUTIONS == 2 + assert config.SKIP == [] + assert config.MAIL_TO == [] + assert config.ITERATIONS_ALL_VMS == 5 + assert config.HEAP_LIMIT == 2097152 + + +def test_read_corrupt_config(): + path = "krun/tests/corrupt.krun" + with pytest.raises(Exception): + Config(path) + + +def test_results_filename(): + empty, example = ".krun", "example.krun" + touch(empty) + touch(example) + empty_config = Config(empty) + example_config = Config(example) + assert empty_config.results_filename() == "_results.json.bz2" + assert example_config.results_filename() == "example_results.json.bz2" + os.unlink(empty) + os.unlink(example) + + +def test_should_skip(): + config = Config("krun/tests/skips.krun") + expected = ["*:PyPy:*", + "*:CPython:*", + "*:Hotspot:*", + "*:Graal:*", + "*:LuaJIT:*", + "*:HHVM:*", + "*:JRubyTruffle:*", + "*:V8:*", + ] + for triplet in expected: + assert config.should_skip(triplet) + assert config.should_skip("nbody:HHVM:default-php") + assert not config.should_skip("nbody:MYVM:default-php") diff --git a/krun/tests/test_entry_point.py b/krun/tests/test_entry_point.py new file mode 100644 index 0000000..745caa2 --- /dev/null +++ b/krun/tests/test_entry_point.py @@ -0,0 +1,10 @@ +from krun import EntryPoint + + +def test_eq(): + ep0 = EntryPoint("test0", subdir="/root/") + ep1 = EntryPoint("test1", subdir="/home/krun/") + assert ep0 == ep0 + assert ep1 == ep1 + assert not ep0 == ep1 + assert not ep0 == list() diff --git a/krun/tests/test_results.py b/krun/tests/test_results.py new file mode 100644 index 0000000..96479c3 --- /dev/null +++ b/krun/tests/test_results.py @@ -0,0 +1,63 @@ +from krun.config import Config +from krun.results import Results +from krun.tests.mocks import MockPlatform, MockMailer + +import os + + +def test_eq(): + results = Results(None, None, + results_file="krun/tests/quick_results.json.bz2") + assert results == results + assert not results == None + assert not results == Results(Config("krun/tests/example.krun"), + MockPlatform(MockMailer())) + + +def test_dump_config(): + """Simulates krun.py --dump-config RESULTS_FILE.json.bz2 + """ + results = Results(None, None, + results_file="krun/tests/quick_results.json.bz2") + with open("krun/tests/quick.krun") as fp: + config = fp.read() + assert config == results.config.text + + +def test_read_results_from_disk(): + config = Config("krun/tests/quick.krun") + results = Results(None, None, + results_file="krun/tests/quick_results.json.bz2") + expected = {u'nbody:CPython:default-python': [[0.022256]], + u'dummy:CPython:default-python': [[1.005115]], + u'nbody:Java:default-java': [[26.002632]], + u'dummy:Java:default-java': [[1.000941]]} + assert results.config == config + assert results.audit[u'uname'] == u'Linux' + assert results.audit[u'debian_version'] == u'jessie/sid' + assert results.data == expected + assert results.starting_temperatures == [4355, 9879] + assert results.eta_estimates == \ + { + u'nbody:CPython:default-python': [0.022256], + u'dummy:CPython:default-python': [1.005115], + u'nbody:Java:default-java': [26.002632], + u'dummy:Java:default-java': [1.000941] + } + + +def test_write_results_to_disk(): + config = Config("krun/tests/example.krun") + out_file = "krun/tests/example_results.json.bz2" + results0 = Results(config, MockPlatform(MockMailer())) + results0.audit = dict() + results0.starting_temperatures = [4355, 9879] + results0.data = {u"dummy:Java:default-java": [[1.000726]]} + results0.eta_estimates = {u"dummy:Java:default-java": [1.1]} + results0.reboots = 5 + results0.error_flag = False + results0.write_to_file() + results1 = Results(config, None, results_file=out_file) + assert results0 == results1 + # Clean-up generated file. + os.unlink(out_file) diff --git a/krun/tests/test_scheduler.py b/krun/tests/test_scheduler.py index f13891e..955a3e5 100644 --- a/krun/tests/test_scheduler.py +++ b/krun/tests/test_scheduler.py @@ -1,10 +1,11 @@ -from krun.tests.mocks import MockMailer, MockPlatform +from krun.audit import Audit +from krun.config import Config +from krun.results import Results from krun.scheduler import mean, ExecutionJob, ExecutionScheduler, JobMissingError +from krun.tests.mocks import MockMailer, MockPlatform import krun.util import os, pytest, subprocess -import bz2 -import json def test_mean_empty(): @@ -21,13 +22,12 @@ def test_mean(): def test_add_del_job(): mailer = MockMailer() - sched = ExecutionScheduler("", "example_test.log", - "example_test.json.bz2", mailer, - MockPlatform(mailer), resume=True, + sched = ExecutionScheduler(Config("krun/tests/example.krun"), mailer, + MockPlatform(mailer), resume=False, reboot=True, dry_run=False, started_by_init=False) assert len(sched) == 0 - sched.add_job(ExecutionJob(sched, None, "CPython", "", "mybench", + sched.add_job(ExecutionJob(sched, "CPython", "", "mybench", "default-python", 1000)) assert len(sched) == 1 sched.remove_job_by_key("mybench:CPython:default-python") @@ -38,22 +38,18 @@ def test_add_del_job(): def test_build_schedule(): mailer = MockMailer() - config = krun.util.read_config('krun/tests/example.krun') - sched = ExecutionScheduler("example.krun", "example_test.log", - "example_test.json.bz2", mailer, - MockPlatform(mailer), resume=True, + sched = ExecutionScheduler(Config("krun/tests/example.krun"), mailer, + MockPlatform(mailer), resume=False, reboot=True, dry_run=True, started_by_init=False) - sched.build_schedule(config, None) + sched.build_schedule() assert len(sched) == 8 - dummy_py = ExecutionJob(sched, config, "CPython", "", "dummy", + dummy_py = ExecutionJob(sched, "CPython", "", "dummy", "default-python", 1000) - dummy_java = ExecutionJob(sched, config, "Java", "", "dummy", - "default-java", 1000) - nbody_py = ExecutionJob(sched, config, "CPython", "", "nbody", + dummy_java = ExecutionJob(sched, "Java", "", "dummy", "default-java", 1000) + nbody_py = ExecutionJob(sched, "CPython", "", "nbody", "default-python", 1000) - nbody_java = ExecutionJob(sched, config, "Java", "", "nbody", - "default-java", 1000) + nbody_java = ExecutionJob(sched, "Java", "", "nbody", "default-java", 1000) assert sched.work_deque.count(dummy_py) == 2 assert sched.work_deque.count(dummy_java) == 2 assert sched.work_deque.count(nbody_py) == 2 @@ -62,78 +58,69 @@ def test_build_schedule(): def test_part_complete_schedule(): mailer = MockMailer() - config = krun.util.read_config('krun/tests/quick.krun') - results_json = krun.util.read_results('krun/tests/quick_results.json.bz2') - sched = ExecutionScheduler("quick.krun", "quick_test.log", - "quick_test.json.bz2", mailer, + sched = ExecutionScheduler(Config("krun/tests/quick.krun"), mailer, MockPlatform(mailer), resume=True, reboot=True, dry_run=True, started_by_init=False) - sched.build_schedule(config, results_json) + sched.build_schedule() assert len(sched) == 0 -def test_eta_dont_agree_with_schedule(): +def test_etas_dont_agree_with_schedule(): """ETAs don't exist for all jobs for which there is iterations data""" mailer = MockMailer() - config = krun.util.read_config('krun/tests/broken_etas.krun') - results_json = krun.util.read_results('krun/tests/broken_etas_results.json.bz2') - sched = ExecutionScheduler("broken_etas.krun", "broken_etas.log", - "broken_etas_results.json.bz2", mailer, - MockPlatform(mailer), resume=True, - reboot=False, dry_run=True, + sched = ExecutionScheduler(Config("krun/tests/broken_etas.krun"), + mailer, + MockPlatform(mailer), + resume=True, reboot=False, dry_run=True, started_by_init=False) try: - sched.build_schedule(config, results_json) + sched.build_schedule() except SystemExit: pass else: - assert(False) # did not exit! + assert False, "Krun did not exit when ETAs failed to tally with results!" def test_run_schedule(monkeypatch): - jso_file = "example_test.json.bz2" + json_file = "krun/tests/example_results.json.bz2" def dummy_shell_cmd(text): pass monkeypatch.setattr(subprocess, 'call', dummy_shell_cmd) monkeypatch.setattr(krun.util, 'run_shell_cmd', dummy_shell_cmd) mailer = MockMailer() - config = krun.util.read_config('krun/tests/example.krun') platform = MockPlatform(mailer) - for vm_name, vm_info in config["VMS"].items(): - vm_info["vm_def"].set_platform(platform) - sched = ExecutionScheduler("krun/tests/example.krun", "example_test.log", - jso_file, mailer, + sched = ExecutionScheduler(Config("krun/tests/example.krun"), + mailer, platform, resume=False, reboot=False, dry_run=True, started_by_init=False) - sched.build_schedule(config, None) + sched.build_schedule() assert len(sched) == 8 sched.run() assert len(sched) == 0 - # Type checks on what the scheduler dumped - with bz2.BZ2File(jso_file, 'rb') as input_file: - jso = json.loads(input_file.read()) - - for k, execs in jso["data"].iteritems(): - assert type(execs) is list - for one_exec in execs: - assert type(one_exec) is list - assert all([type(x) is float for x in one_exec]) + results = Results(Config("krun/tests/example.krun"), + MockPlatform(MockMailer()), + results_file=json_file) + for k, execs in results.data.iteritems(): + assert type(execs) is list + for one_exec in execs: + assert type(one_exec) is list + assert all([type(x) is float for x in one_exec]) - for k, execs in jso["eta_estimates"].iteritems(): - assert type(execs) is list - assert all([type(x) is float for x in execs]) + for k, execs in results.eta_estimates.iteritems(): + assert type(execs) is list + assert all([type(x) is float for x in execs]) - assert type(jso["starting_temperatures"]) is list - assert type(jso["reboots"]) is int - assert type(jso["audit"]) is dict - assert type(jso["config"]) is unicode - assert type(jso["error_flag"]) is bool + assert type(results.starting_temperatures) is list + assert type(results.reboots) is int + assert type(results.audit) is type(Audit(dict())) + assert type(results.config) is type(Config()) + assert type(results.error_flag) is bool - os.unlink(jso_file) + os.unlink(json_file) def test_run_schedule_reboot(monkeypatch): @@ -141,22 +128,19 @@ def dummy_shell_cmd(text): pass def dummy_execv(text, lst): pass - monkeypatch.setattr(os, 'execv', dummy_execv) - monkeypatch.setattr(subprocess, 'call', dummy_shell_cmd) - monkeypatch.setattr(krun.util, 'run_shell_cmd', dummy_shell_cmd) + monkeypatch.setattr(os, "execv", dummy_execv) + monkeypatch.setattr(subprocess, "call", dummy_shell_cmd) + monkeypatch.setattr(krun.util, "run_shell_cmd", dummy_shell_cmd) mailer = MockMailer() - config = krun.util.read_config('krun/tests/example.krun') platform = MockPlatform(mailer) - for vm_name, vm_info in config["VMS"].items(): - vm_info["vm_def"].set_platform(platform) - sched = ExecutionScheduler("krun/tests/example.krun", "example_test.log", - "example_test.json.bz2", mailer, - platform, resume=True, + sched = ExecutionScheduler(Config("krun/tests/example.krun"), + mailer, + platform, resume=False, reboot=True, dry_run=True, started_by_init=True) - sched.build_schedule(config, None) + sched.build_schedule() assert len(sched) == 8 with pytest.raises(AssertionError): sched.run() assert len(sched) == 7 - os.unlink("example_test.json.bz2") + os.unlink("krun/tests/example_results.json.bz2") diff --git a/krun/tests/test_util.py b/krun/tests/test_util.py index 3ab10dd..277f3e2 100644 --- a/krun/tests/test_util.py +++ b/krun/tests/test_util.py @@ -1,61 +1,12 @@ -from krun import LOGFILE_FILENAME_TIME_FORMAT -from krun.util import (should_skip, format_raw_exec_results, output_name, - log_and_mail, log_name, fatal, read_config, - run_shell_cmd, read_results, dump_results, +from krun.util import (format_raw_exec_results, + log_and_mail, fatal, check_and_parse_execution_results, - audits_same_platform, ExecutionFailed) + run_shell_cmd, + ExecutionFailed) from krun.tests.mocks import MockMailer -import bz2, json, os, pytest, time - - -def test_should_skip(): - config = dict([("SKIP", ["*:PyPy:*", - "*:CPython:*", - "*:Hotspot:*", - "*:Graal:*", - "*:LuaJIT:*", - "*:HHVM:*", - "*:JRubyTruffle:*", - "*:V8:*", - ])]) - assert should_skip(config, "nbody:HHVM:default-php") - assert not should_skip(dict([("SKIP", [])]), "nbody:HHVM:default-php") - - -def test_read_config(): - path = "examples/example.krun" - config = read_config(path) - assert config is not None - assert config["BENCHMARKS"] == {"dummy": 1000, "nbody": 1000} - assert config["N_EXECUTIONS"] == 2 - assert config["SKIP"] == [] - assert config["MAIL_TO"] == [] - assert config["ITERATIONS_ALL_VMS"] == 5 - assert config["HEAP_LIMIT"] == 2097152 - - -def test_read_corrupt_config(): - path = "krun/tests/corrupt.krun" - with pytest.raises(Exception): - _ = read_config(path) - - -def test_output_name(): - assert output_name(".krun") == "_results.json.bz2" - assert output_name("example.krun") == "example_results.json.bz2" - - -def test_log_name(monkeypatch): - tstamp = time.strftime(LOGFILE_FILENAME_TIME_FORMAT) - assert log_name("example.krun", False) == "example" + "_" + tstamp + ".log" - assert log_name(".krun", False) == "_" + tstamp + ".log" - def mock_mtime(path): - return 1445964109.9363003 - monkeypatch.setattr(os.path, 'getmtime', mock_mtime) - tstamp = '20151027_164149' - assert log_name("example.krun", True) == "example" + "_" + tstamp + ".log" - assert log_name(".krun", True) == "_" + tstamp + ".log" +import json +import pytest def test_fatal(capsys): @@ -101,62 +52,6 @@ def test_run_shell_cmd_fatal(capsys): assert out == "" -def test_read_results(): - results = read_results('krun/tests/quick_results.json.bz2') - expected = {u'nbody:CPython:default-python': [[0.022256]], - u'dummy:CPython:default-python': [[1.005115]], - u'nbody:Java:default-java': [[26.002632]], - u'dummy:Java:default-java': [[1.000941]]} - with open('krun/tests/quick.krun', 'rb') as config_fp: - config = config_fp.read() - assert results['config'] == config - assert results['audit']['uname'] == u'Linux' - assert results['audit']['debian_version'] == u'jessie/sid' - assert results['data'] == expected - assert results['starting_temperatures'] == [4355, 9879] - assert results['eta_estimates'] == \ - { - u'nbody:CPython:default-python': [0.022256], - u'dummy:CPython:default-python': [1.005115], - u'nbody:Java:default-java': [26.002632], - u'dummy:Java:default-java': [1.000941] - } - - -def test_dump_results(): - - class DummyPlatform(object): - audit = 'example audit (py.test)' - starting_temperatures = [4355, 9879] - - class DummyExecutionScheduler(object): - platform = DummyPlatform() - out_file = output_name("krun/tests/example.krun") - results = {'dummy:Java:default-java': [[1.000726]]} - nreboots = 5 - eta_estimates = {'dummy:Java:default-java': [1.1]} - error_flag = False - config_file = 'krun/tests/example.krun' - - dummy_sched = DummyExecutionScheduler() - dump_results(dummy_sched) - - with open(dummy_sched.config_file, 'r') as config_fp: - config = config_fp.read() - with bz2.BZ2File(dummy_sched.out_file, 'rb') as input_file: - dumped_results = json.loads(input_file.read()) - assert dumped_results['audit'] == dummy_sched.platform.audit - assert dumped_results['starting_temperatures'] == \ - dummy_sched.platform.starting_temperatures - assert dumped_results['config'] == config - assert dumped_results['data'] == dummy_sched.results - assert dumped_results['reboots'] == dummy_sched.nreboots - assert dumped_results['eta_estimates'] == \ - dummy_sched.eta_estimates - assert dumped_results['error_flag'] == dummy_sched.error_flag - os.unlink(dummy_sched.out_file) # Clean-up generated file. - - def test_check_and_parse_execution_results(): stdout = "[0.000403]" stderr = "[iterations_runner.py] iteration 1/1" @@ -192,20 +87,3 @@ def test_check_and_parse_execution_results(): -------------------------------------------------- """ assert excinfo.value.message == expected - - -def test_audit_compare(): - audit0 = dict([("cpuinfo", u"processor\t: 0\nvendor_id\t: GenuineIntel"), - ("uname", u"Linux"), - ("debian_version", u"jessie/sid"), - ("packages", u"1:1.2.8.dfsg-2ubuntu1"), - ("dmesg", u"")]) - audit1 = dict([("cpuinfo", u"processor\t: 0\nvendor_id\t: GenuineIntel"), - ("uname", u"BSD"), - ("packages", u"1:1.2.8.dfsg-2ubuntu1"), - ("dmesg", u"")]) - assert audits_same_platform(audit0, audit0) - assert audits_same_platform(audit1, audit1) - assert not audits_same_platform([], []) - assert not audits_same_platform(audit0, audit1) - assert not audits_same_platform(audit1, audit0) diff --git a/krun/util.py b/krun/util.py index a7b55b3..57681a4 100644 --- a/krun/util.py +++ b/krun/util.py @@ -1,12 +1,7 @@ -import bz2 # decent enough compression with Python 2.7 compatibility. import json -import os.path import sys -import time -from collections import OrderedDict from subprocess import Popen, PIPE from logging import error -from krun import LOGFILE_FILENAME_TIME_FORMAT FLOAT_FORMAT = ".6f" @@ -14,55 +9,6 @@ class ExecutionFailed(Exception): pass -def should_skip(config, this_key): - skips = config["SKIP"] - - for skip_key in skips: - skip_elems = skip_key.split(":") - this_elems = this_key.split(":") - - # should be triples of: bench * vm * variant - assert len(skip_elems) == 3 and len(this_elems) == 3 - - for i in range(3): - if skip_elems[i] == "*": - this_elems[i] = "*" - - if skip_elems == this_elems: - return True # skip - - return False - - -def read_config(path): - assert path.endswith(".krun") - dct = {} - try: - execfile(path, dct) - except Exception as e: - error("error importing config file:\n%s" % str(e)) - raise - - return dct - - -def output_name(config_path): - """Makes a result file name based upon the config file name.""" - - assert config_path.endswith(".krun") - return config_path[:-5] + "_results.json.bz2" - - -def log_name(config_path, resume=False): - assert config_path.endswith(".krun") - if resume: - config_mtime = time.gmtime(os.path.getmtime(config_path)) - tstamp = time.strftime(LOGFILE_FILENAME_TIME_FORMAT, config_mtime) - else: - tstamp = time.strftime(LOGFILE_FILENAME_TIME_FORMAT) - return config_path[:-5] + "_%s.log" % tstamp - - def fatal(msg): error(msg) sys.exit(1) @@ -93,32 +39,6 @@ def run_shell_cmd(cmd, failure_fatal=True): return stdout.strip(), stderr.strip(), rc -def dump_results(sched): - """Dump results (and a few other bits) into a bzip2 json file.""" - with open(sched.config_file, "r") as f: - config_text = f.read() - - to_write = { - "config": config_text, - "data": sched.results, - "audit": sched.platform.audit, - "reboots": sched.nreboots, - "starting_temperatures": sched.platform.starting_temperatures, - "eta_estimates": sched.eta_estimates, - "error_flag": sched.error_flag, - } - - with bz2.BZ2File(sched.out_file, "w") as f: - f.write(json.dumps(to_write, indent=1, sort_keys=True)) - - -def read_results(results_file): - results = None - with bz2.BZ2File(results_file, "rb") as f: - results = json.loads(f.read()) - return results - - def check_and_parse_execution_results(stdout, stderr, rc): json_exn = None try: @@ -138,29 +58,3 @@ def check_and_parse_execution_results(stdout, stderr, rc): raise ExecutionFailed(err_s) return iterations_results - - -def audits_same_platform(audit0, audit1): - """Check whether two platform audits are from identical machines. - A machine audit is a dictionary with the following keys: - - * cpuinfo (Linux only) - * packages (Debian-based systems only) - * debian_version (Debian-based systems only) - * uname (all platforms) - * dmesg (all platforms) - - Platform information may be Unicode. - """ - if ("uname" not in audit0) or ("uname" not in audit1): - return False - return audit0["uname"] == audit1["uname"] - -def dump_audit(audit): - s = "" - # important that the sections are sorted, for diffing - for key, text in OrderedDict(sorted(audit.iteritems())).iteritems(): - s += "Audit Section: %s" % key + "\n" - s += "#" * 78 + "\n\n" - s += text + "\n\n" - return s diff --git a/krun/vm_defs.py b/krun/vm_defs.py index 7e8a3f0..0ba4e93 100644 --- a/krun/vm_defs.py +++ b/krun/vm_defs.py @@ -219,6 +219,9 @@ def run_vm_sanity_check(self, entry_point): "return code: %s\nstdout:%s\nstderr: %s" % (entry_point.target, rc, stdout, stderr)) + def __eq__(self, other): + return isinstance(other, self.__class__) + class NativeCodeVMDef(BaseVMDef): """Not really a "VM definition" at all. Runs native code."""