From 14db4eb011ca1b7477ecb291be1a49f814d3c2b4 Mon Sep 17 00:00:00 2001 From: Kefu Chai Date: Sat, 4 Aug 2018 00:25:41 +0800 Subject: [PATCH 1/3] suite: do not bail out if results.log is not around Signed-off-by: Kefu Chai --- teuthology/suite/__init__.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/teuthology/suite/__init__.py b/teuthology/suite/__init__.py index 52882962ad..60274b4bce 100644 --- a/teuthology/suite/__init__.py +++ b/teuthology/suite/__init__.py @@ -111,7 +111,10 @@ def get_rerun_filters(name, statuses): def get_rerun_conf(conf): reporter = ResultsReporter() - subset, seed = reporter.get_rerun_conf(conf.rerun) + try: + subset, seed = reporter.get_rerun_conf(conf.rerun) + except IOError: + return None, None if seed is None: return conf.subset, conf.seed if conf.seed < 0: From 753c75ee3980e5843d716e9c69cc9aae8cfea952 Mon Sep 17 00:00:00 2001 From: Kefu Chai Date: Fri, 24 Aug 2018 19:40:14 +0800 Subject: [PATCH 2/3] suite.run: extract write_result() out of prepare_and_schedule() so it's more modular. Signed-off-by: Kefu Chai --- teuthology/suite/run.py | 40 +++++++++++++++++++++------------------- 1 file changed, 21 insertions(+), 19 deletions(-) diff --git a/teuthology/suite/run.py b/teuthology/suite/run.py index 8abca0fc6b..8de984bdba 100644 --- a/teuthology/suite/run.py +++ b/teuthology/suite/run.py @@ -291,6 +291,26 @@ def build_base_args(self): base_args.extend(['--owner', self.args.owner]) return base_args + def write_result(self): + arg = copy.deepcopy(self.base_args) + arg.append('--last-in-suite') + if self.base_config.email: + arg.extend(['--email', self.base_config.email]) + if self.args.subset: + subset = '/'.join(str(i) for i in self.args.subset) + arg.extend(['--subset', subset]) + arg.extend(['--seed', str(self.args.seed)]) + if self.args.timeout: + arg.extend(['--timeout', self.args.timeout]) + util.teuthology_schedule( + args=arg, + dry_run=self.args.dry_run, + verbose=self.args.verbose, + log_prefix="Results: ") + results_url = get_results_url(self.base_config.name) + if results_url: + log.info("Test results viewable at %s", results_url) + def prepare_and_schedule(self): """ Puts together some "base arguments" with which to execute @@ -309,25 +329,7 @@ def prepare_and_schedule(self): num_jobs = self.schedule_suite() if num_jobs: - arg = copy.deepcopy(self.base_args) - arg.append('--last-in-suite') - if self.base_config.email: - arg.extend(['--email', self.base_config.email]) - if self.args.subset: - subset = '/'.join(str(i) for i in self.args.subset) - arg.extend(['--subset', subset]) - arg.extend(['--seed', str(self.args.seed)]) - if self.args.timeout: - arg.extend(['--timeout', self.args.timeout]) - util.teuthology_schedule( - args=arg, - dry_run=self.args.dry_run, - verbose=self.args.verbose, - log_prefix="Results: ", - ) - results_url = get_results_url(self.base_config.name) - if results_url: - log.info("Test results viewable at %s", results_url) + self.write_result() def collect_jobs(self, arch, configs, newest=False): jobs_to_schedule = [] From d488b9bda577781429e8d28d7802b144b1e37dce Mon Sep 17 00:00:00 2001 From: Kefu Chai Date: Fri, 24 Aug 2018 21:15:31 +0800 Subject: [PATCH 3/3] suite/run,schedule,result: write rerun memo as the first job in suite so we don't need to wait for the job to write result to for rerunning the test suite. without this change, the "result" is normally the last job in the suite to be scheduled, so it's likely we will not have the results.log until the suite is almost completed. afer this change, a "first-in-suite" job is scheduled as the first job to note down the subset and seed to run the suite. Signed-off-by: Kefu Chai --- scripts/schedule.py | 3 +++ teuthology/results.py | 19 ++++++++++++------- teuthology/schedule.py | 11 ++++++++++- teuthology/suite/run.py | 24 ++++++++++++++++++++---- teuthology/suite/test/test_run_.py | 5 +++++ teuthology/test/test_schedule.py | 2 ++ teuthology/worker.py | 8 ++++---- 7 files changed, 56 insertions(+), 16 deletions(-) diff --git a/scripts/schedule.py b/scripts/schedule.py index 35a4f70ae9..feeafd770e 100644 --- a/scripts/schedule.py +++ b/scripts/schedule.py @@ -26,6 +26,9 @@ -N , --num Number of times to run/queue the job [default: 1] + --first-in-suite Mark the first job in a suite so suite + can note down the rerun-related info + [default: False] --last-in-suite Mark the last job in a suite so suite post-processing can be run [default: False] diff --git a/teuthology/results.py b/teuthology/results.py index f652a2d9b0..c5baa91b27 100644 --- a/teuthology/results.py +++ b/teuthology/results.py @@ -28,21 +28,26 @@ def main(args): teuthology.setup_log_file(log_path) try: - results(args['--archive-dir'], args['--name'], args['--email'], - int(args['--timeout']), args['--dry-run'], - args['--subset'], args['--seed']) + if args['--seed']: + note_rerun_params(args['--subset'], args['--seed']) + else: + results(args['--archive-dir'], args['--name'], args['--email'], + int(args['--timeout']), args['--dry-run']) except Exception: - log.exception('error generating results') + log.exception('error generating memo/results') raise -def results(archive_dir, name, email, timeout, dry_run, subset, seed): - starttime = time.time() - +def note_rerun_params(subset, seed): if subset: log.info('subset: %r', subset) if seed: log.info('seed: %r', seed) + + +def results(archive_dir, name, email, timeout, dry_run): + starttime = time.time() + if timeout: log.info('Waiting up to %d seconds for tests to finish...', timeout) diff --git a/teuthology/schedule.py b/teuthology/schedule.py index d7762c268c..8305aa10a2 100644 --- a/teuthology/schedule.py +++ b/teuthology/schedule.py @@ -7,8 +7,16 @@ def main(args): + if not args['--first-in-suite']: + first_job_args = ['subset', 'seed'] + for arg in first_job_args: + opt = '--{arg}'.format(arg=arg) + msg_fmt = '{opt} is only applicable to the first job in a suite' + if args[opt]: + raise ValueError(msg_fmt.format(opt=opt)) + if not args['--last-in-suite']: - last_job_args = ['email', 'timeout', 'subset', 'seed'] + last_job_args = ['email', 'timeout'] for arg in last_job_args: opt = '--{arg}'.format(arg=arg) msg_fmt = '{opt} is only applicable to the last job in a suite' @@ -43,6 +51,7 @@ def build_config(args): job_config = dict( name=args['--name'], + first_in_suite=args['--first-in-suite'], last_in_suite=args['--last-in-suite'], email=args['--email'], description=args['--description'], diff --git a/teuthology/suite/run.py b/teuthology/suite/run.py index 8de984bdba..5c01e39ceb 100644 --- a/teuthology/suite/run.py +++ b/teuthology/suite/run.py @@ -291,15 +291,26 @@ def build_base_args(self): base_args.extend(['--owner', self.args.owner]) return base_args + + def write_rerun_memo(self): + args = copy.deepcopy(self.base_args) + args.append('--first-in-suite') + if self.args.subset: + subset = '/'.join(str(i) for i in self.args.subset) + args.extend(['--subset', subset]) + args.extend(['--seed', str(self.args.seed)]) + util.teuthology_schedule( + args=args, + dry_run=self.args.dry_run, + verbose=self.args.verbose, + log_prefix="Memo: ") + + def write_result(self): arg = copy.deepcopy(self.base_args) arg.append('--last-in-suite') if self.base_config.email: arg.extend(['--email', self.base_config.email]) - if self.args.subset: - subset = '/'.join(str(i) for i in self.args.subset) - arg.extend(['--subset', subset]) - arg.extend(['--seed', str(self.args.seed)]) if self.args.timeout: arg.extend(['--timeout', self.args.timeout]) util.teuthology_schedule( @@ -311,6 +322,7 @@ def write_result(self): if results_url: log.info("Test results viewable at %s", results_url) + def prepare_and_schedule(self): """ Puts together some "base arguments" with which to execute @@ -526,6 +538,10 @@ def schedule_suite(self): with open(base_yaml_path, 'w+b') as base_yaml: base_yaml.write(str(self.base_config)) + + if jobs_to_schedule: + self.write_rerun_memo() + self.schedule_jobs(jobs_missing_packages, jobs_to_schedule, name) os.remove(base_yaml_path) diff --git a/teuthology/suite/test/test_run_.py b/teuthology/suite/test/test_run_.py index 363781e236..3251356b06 100644 --- a/teuthology/suite/test/test_run_.py +++ b/teuthology/suite/test/test_run_.py @@ -196,6 +196,7 @@ def setup(self): self.args = YamlConfig.from_dict(self.args_dict) @patch('teuthology.suite.run.Run.schedule_jobs') + @patch('teuthology.suite.run.Run.write_rerun_memo') @patch('teuthology.suite.util.has_packages_for_distro') @patch('teuthology.suite.util.get_package_versions') @patch('teuthology.suite.util.get_install_task_flavor') @@ -216,6 +217,7 @@ def test_successful_schedule( m_get_install_task_flavor, m_get_package_versions, m_has_packages_for_distro, + m_write_rerun_memo, m_schedule_jobs, ): m_get_arch.return_value = 'x86_64' @@ -266,6 +268,7 @@ def test_successful_schedule( m_schedule_jobs.assert_has_calls( [call([], [expected_job], runobj.name)], ) + m_write_rerun_memo.assert_called_once_with() @patch('teuthology.suite.util.find_git_parent') @patch('teuthology.suite.run.Run.schedule_jobs') @@ -323,6 +326,7 @@ def test_newest_failure( @patch('teuthology.suite.util.find_git_parent') @patch('teuthology.suite.run.Run.schedule_jobs') + @patch('teuthology.suite.run.Run.write_rerun_memo') @patch('teuthology.suite.util.has_packages_for_distro') @patch('teuthology.suite.util.get_package_versions') @patch('teuthology.suite.util.get_install_task_flavor') @@ -343,6 +347,7 @@ def test_newest_success( m_get_install_task_flavor, m_get_package_versions, m_has_packages_for_distro, + m_write_rerun_memo, m_schedule_jobs, m_find_git_parent, ): diff --git a/teuthology/test/test_schedule.py b/teuthology/test/test_schedule.py index fef0d4e019..4c52eed768 100644 --- a/teuthology/test/test_schedule.py +++ b/teuthology/test/test_schedule.py @@ -8,6 +8,7 @@ class TestSchedule(object): '--owner': 'OWNER', '--description': 'DESC', '--email': 'EMAIL', + '--first-in-suite': False, '--last-in-suite': True, '--name': 'NAME', '--worker': 'tala', @@ -22,6 +23,7 @@ def test_basic(self): expected = { 'description': 'DESC', 'email': 'EMAIL', + 'first_in_suite': False, 'last_in_suite': True, 'machine_type': 'tala', 'name': 'NAME', diff --git a/teuthology/worker.py b/teuthology/worker.py index 4b76de9605..351f47e97b 100644 --- a/teuthology/worker.py +++ b/teuthology/worker.py @@ -190,10 +190,10 @@ def prep_job(job_config, log_file_path, archive_dir): def run_job(job_config, teuth_bin_path, archive_dir, verbose): safe_archive = safepath.munge(job_config['name']) - if job_config.get('last_in_suite'): + if job_config.get('first_in_suite') or job_config.get('last_in_suite'): if teuth_config.results_server: report.try_delete_jobs(job_config['name'], job_config['job_id']) - log.info('Generating results for %s', job_config['name']) + log.info('Generating memo/results for %s', job_config['name']) args = [ os.path.join(teuth_bin_path, 'teuthology-results'), '--timeout', @@ -203,11 +203,11 @@ def run_job(job_config, teuth_bin_path, archive_dir, verbose): os.path.join(archive_dir, safe_archive), '--name', job_config['name'], - '--seed', - job_config['seed'], ] if job_config.get('email'): args.extend(['--email', job_config['email']]) + if job_config.get('seed'): + args.extend(['--seed', job_config['seed']]) if job_config.get('subset'): args.extend(['--subset', job_config['subset']]) # Execute teuthology-results, passing 'preexec_fn=os.setpgrp' to