Merge pull request #134 from ceph/wip-6546

Fix race condition between tests starting and teuthology-results being run

teuthology/report.py

@@ -115,6 +115,19 @@ def jobs_for_run(self, run_name):
                 jobs[job_id] = job_dir
         return jobs
 
+    def running_jobs_for_run(self, run_name):
+        """
+        Like jobs_for_run(), but only returns jobs with no summary.yaml
+
+        :param run_name: The name of the run.
+        :returns:        A dict like: {'1': '/path/to/1', '2': 'path/to/2'}
+        """
+        jobs = self.jobs_for_run(run_name)
+        for name in jobs.keys():
+            if not os.path.exists(os.path.join(jobs[name], 'summary.yaml')):
+                jobs.pop(name)
+        return jobs
+
     @property
     def all_runs(self):
         """

teuthology/results.py

@@ -10,6 +10,7 @@
 import teuthology
 from teuthology import misc
 from teuthology import suite
+from .report import ResultsSerializer
 
 log = logging.getLogger(__name__)
 
@@ -40,26 +41,17 @@ def main(args):
 
 
 def results(args):
-    running_tests = [
-        f for f in sorted(os.listdir(args.archive_dir))
-        if not f.startswith('.')
-        and os.path.isdir(os.path.join(args.archive_dir, f))
-        and not os.path.exists(os.path.join(
-            args.archive_dir, f, 'summary.yaml'))
-    ]
+    archive_base = os.path.split(args.archive_dir, args.name)
+    serializer = ResultsSerializer(archive_base)
     starttime = time.time()
+
     log.info('Waiting up to %d seconds for tests to finish...', args.timeout)
-    while running_tests and args.timeout > 0:
-        if os.path.exists(os.path.join(
-                args.archive_dir,
-                running_tests[-1], 'summary.yaml')):
-            running_tests.pop()
-        else:
-            if time.time() - starttime > args.timeout:
-                log.warn('test(s) did not finish before timeout of %d seconds',
-                         args.timeout)
-                break
-            time.sleep(10)
+    while serializer.running_jobs_for_run(args.name) and args.timeout > 0:
+        if time.time() - starttime > args.timeout:
+            log.warn('test(s) did not finish before timeout of %d seconds',
+                     args.timeout)
+            break
+        time.sleep(10)
     log.info('Tests finished! gathering results...')
 
     (subject, body) = build_email_body(args.name, args.archive_dir,

teuthology/test/fake_archive.py

@@ -79,7 +79,7 @@ def populate_archive(self, run_name, jobs):
                 with file(summary_path, 'w') as yfile:
                     yaml.safe_dump(job['summary'], yfile)
 
-    def create_fake_run(self, run_name, job_count, yaml_path):
+    def create_fake_run(self, run_name, job_count, yaml_path, num_hung=0):
         """
         Creates a fake run using run_name. Uses the YAML specified for each
         job's config.yaml
@@ -89,8 +89,13 @@ def create_fake_run(self, run_name, job_count, yaml_path):
         assert os.path.exists(yaml_path)
         assert job_count > 0
         jobs = []
+        made_hung = 0
         for i in range(job_count):
-            jobs.append(self.get_random_metadata(run_name))
+            if made_hung < num_hung:
+                jobs.append(self.get_random_metadata(run_name, hung=True))
+                made_hung += 1
+            else:
+                jobs.append(self.get_random_metadata(run_name, hung=False))
             #job_config = yaml.safe_load(yaml_path)
         self.populate_archive(run_name, jobs)
         for job in jobs:

teuthology/test/test_report.py

@@ -45,6 +45,17 @@ def test_jobs_for_run(self):
         got_jobs = self.reporter.serializer.jobs_for_run(run_name)
         assert sorted(job_ids) == sorted(got_jobs.keys())
 
+    def test_running_jobs_for_run(self):
+        run_name = "test_jobs_for_run"
+        yaml_path = "examples/3node_ceph.yaml"
+        job_count = 10
+        num_hung = 3
+        self.archive.create_fake_run(run_name, job_count, yaml_path,
+                                     num_hung=num_hung)
+
+        got_jobs = self.reporter.serializer.running_jobs_for_run(run_name)
+        assert len(got_jobs) == job_count - num_hung
+
     def test_json_for_job(self):
         run_name = "test_json_for_job"
         yaml_path = "examples/3node_ceph.yaml"