artifact collection for bazel targets

Summary: This diff has Changes look at *.bazel.xml and appropriately create Targets for them. This involves a new model, BazelTarget, and a new artifact handler, BazelTargetHandler.

Test Plan: unit tests

Reviewers: anupc

Reviewed By: anupc

Subscribers: changesbot, kylec

Differential Revision: https://tails.corp.dropbox.com/D224599

changes/artifacts/bazel_target.py

@@ -0,0 +1,66 @@
+import os
+
+from changes.artifacts.xunit import XunitHandler
+from changes.config import db
+from changes.constants import Status
+from changes.db.utils import get_or_create
+from changes.models.bazeltarget import BazelTarget
+from changes.models.test import TestCase
+from changes.models.testresult import TestResultManager
+from changes.storage.artifactstore import ARTIFACTSTORE_PREFIX
+from changes.utils.agg import aggregate_result
+
+
+class BazelTargetHandler(XunitHandler):
+    FILENAMES = ('test.bazel.xml',)
+
+    def process(self, fp, artifact):
+        target_name = self._get_target_name(artifact)
+        target, _ = get_or_create(BazelTarget, where={
+            'step_id': self.step.id,
+            'job_id': self.step.job.id,
+            'name': target_name,
+        })
+        test_suites = self.get_test_suites(fp)
+        tests = self.aggregate_tests_from_suites(test_suites)
+        manager = TestResultManager(self.step, artifact)
+        manager.save(tests)
+
+        # add all tests to target
+        for test in tests:
+            test_case = TestCase.query.filter(
+                TestCase.step == self.step,
+                TestCase.name_sha == test.name_sha,
+            ).limit(1).first()
+            target.tests.append(test_case)
+
+        # update target metadata
+        # TODO handle multiple files per target, i.e. sharding and running multiple times
+        target.status = Status.finished
+        target.result = aggregate_result([t.result for t in target.tests])
+        duration = 0
+        for t in test_suites:
+            if t.duration is None:
+                duration = None
+                break
+            duration += t.duration
+        target.duration = duration
+        target.date_created = min([t.date_created for t in test_suites])
+        db.session.add(target)
+        db.session.commit()
+        return tests
+
+    def _get_target_name(self, artifact):
+        """Given an artifact, return the target name relative to the root
+        of the repo.
+
+        Essentially, we want to go from the artifact name of
+        {artifact_store_prefix}foo/bar/baz/test.bazel.xml to
+        //foo/bar:baz
+        """
+        assert artifact.name.startswith(ARTIFACTSTORE_PREFIX)
+        path = artifact.name[len(ARTIFACTSTORE_PREFIX):]
+        dirname = os.path.dirname(path)
+        dirname, target = os.path.split(dirname)
+        target = dirname + ':' + target
+        return '//' + target.lstrip('/')

changes/artifacts/xunit.py

@@ -18,7 +18,7 @@
 
 
 class XunitHandler(ArtifactHandler):
-    FILENAMES = ('xunit.xml', 'junit.xml', 'nosetests.xml', '*.xunit.xml', '*.junit.xml', '*.nosetests.xml', '*.bazel.xml')
+    FILENAMES = ('xunit.xml', 'junit.xml', 'nosetests.xml', '*.xunit.xml', '*.junit.xml', '*.nosetests.xml')
     logger = logging.getLogger('xunit')
 
     def process(self, fp, artifact):

changes/buildsteps/default.py

@@ -9,6 +9,7 @@
 from typing import Dict, List, Optional  # NOQA
 
 from changes.artifacts.analytics_json import AnalyticsJsonHandler
+from changes.artifacts.bazel_target import BazelTargetHandler
 from changes.artifacts.coverage import CoverageHandler
 from changes.artifacts.manager import Manager
 from changes.artifacts.xunit import XunitHandler
@@ -546,7 +547,7 @@ def get_allocation_command(self, jobstep):
         ))
 
     def get_artifact_manager(self, jobstep):
-        return Manager([CoverageHandler, XunitHandler, AnalyticsJsonHandler])
+        return Manager([CoverageHandler, BazelTargetHandler, XunitHandler, AnalyticsJsonHandler])
 
     def prefer_artifactstore(self):
         return self.debug_config.get('prefer_artifactstore', True)

changes/models/bazeltarget.py

@@ -0,0 +1,37 @@
+from __future__ import absolute_import, division
+
+import uuid
+
+from datetime import datetime
+from sqlalchemy import Column, DateTime, ForeignKey, Text, Integer
+from sqlalchemy.orm import backref, relationship
+
+from changes.config import db
+from changes.constants import Result, Status
+from changes.db.types.enum import Enum
+from changes.db.types.guid import GUID
+
+
+class BazelTarget(db.Model):
+    __tablename__ = 'bazeltarget'
+    id = Column(GUID, nullable=False, primary_key=True, default=uuid.uuid4)
+    step_id = Column(GUID, ForeignKey('jobstep.id', ondelete="CASCADE"), nullable=False)
+    job_id = Column(GUID, ForeignKey('job.id', ondelete="CASCADE"), nullable=False)
+    name = Column(Text, nullable=False)
+    status = Column(Enum(Status), nullable=False, default=Status.unknown)
+    result = Column(Enum(Result), default=Result.unknown, nullable=False)
+    duration = Column(Integer, default=0)
+    date_created = Column(DateTime, default=datetime.utcnow, nullable=False)
+
+    tests = relationship('TestCase', backref=backref('target'))
+
+    def __init__(self, **kwargs):
+        super(BazelTarget, self).__init__(**kwargs)
+        if self.id is None:
+            self.id = uuid.uuid4()
+        if self.result is None:
+            self.result = Result.unknown
+        if self.status is None:
+            self.status = Status.unknown
+        if self.date_created is None:
+            self.date_created = datetime.utcnow()

changes/models/job.py

@@ -60,6 +60,8 @@ class Job(db.Model):
     project = relationship('Project')
     source = relationship('Source')
 
+    targets = relationship('BazelTarget', backref=backref('job'))
+
     __repr__ = model_repr('label', 'target')
 
     def __init__(self, **kwargs):

changes/models/jobstep.py

@@ -69,6 +69,7 @@ class JobStep(db.Model):
     project = relationship('Project')
     node = relationship('Node')
     phase = relationship('JobPhase', backref=backref('steps', order_by='JobStep.date_started'))
+    targets = relationship('BazelTarget', backref=backref('step'))
 
     __repr__ = model_repr('label')
 

changes/models/test.py

@@ -41,6 +41,7 @@ class TestCase(db.Model):
     job_id = Column(GUID, ForeignKey('job.id', ondelete="CASCADE"), nullable=False)
     project_id = Column(GUID, ForeignKey('project.id', ondelete="CASCADE"), nullable=False)
     step_id = Column(GUID, ForeignKey('jobstep.id', ondelete="CASCADE"))
+    target_id = Column(GUID, ForeignKey('bazeltarget.id', ondelete='CASCADE'), nullable=True)
     name_sha = Column('label_sha', String(40), nullable=False)
     name = Column(Text, nullable=False)
     _package = Column('package', Text, nullable=True)

changes/testutils/fixtures.py

@@ -37,6 +37,7 @@
 
 __all__ = ('Fixtures', 'SAMPLE_COVERAGE', 'SAMPLE_DIFF_BYTES', 'SAMPLE_DIFF', 'SAMPLE_XUNIT',
            'SAMPLE_XUNIT_DOUBLE_CASES', 'SAMPLE_XUNIT_MULTIPLE_SUITES',
+           'SAMPLE_XUNIT_MULTIPLE_SUITES_COMPLETE_TIME',
            'SAMPLE_XUNIT_TESTARTIFACTS')
 
 
@@ -150,6 +151,40 @@
     </testsuite>
 </testsuites>"""
 
+# same as SAMPLE_XUNIT_MULTIPLE_SUITES, but with complete timing information and passing tests
+SAMPLE_XUNIT_MULTIPLE_SUITES_COMPLETE_TIME = """<?xml version="1.0" encoding="utf-8"?>
+<testsuites>
+    <testsuite errors="0" failures="0" skips="0" tests="0" time="0.5">
+        <testcase classname="" name="tests.test_report" time="0" owner="foo">
+        </testcase>
+        <testcase classname="tests.test_report.ParseTestResultsTest" name="test_simple" time="0.00165796279907" rerun="1"/>
+        <testcase classname="test_simple.SampleTest" name="test_falsehood" time="0.50" rerun="3">
+            <system-out>Running SampleTest</system-out>
+            <system-out>Running SampleTest</system-out>
+        </testcase>
+    </testsuite>
+    <testsuite errors="0" failures="0" name="suite2" skips="0" tests="0" time="0.077">
+        <testcase classname="" name="tests2.test_report" time="0" owner="foo">
+        </testcase>
+        <testcase classname="tests2.test_report.ParseTestResultsTest" name="test_simple" time="0.00165796279907" rerun="1"/>
+        <testcase classname="test_simple.SampleTest" name="test_falsehood" time="0.50" rerun="3">
+            <system-out>Running SampleTest</system-out>
+            <system-out>Running SampleTest</system-out>
+        </testcase>
+    </testsuite>
+    <testsuite errors="0" failures="0" name="" skips="0" tests="0" time="1">
+        <testcase classname="" name="tests3.test_report" owner="foo">
+        </testcase>
+        <testcase classname="" name="tests3.test_report" owner="foo">
+        </testcase>
+        <testcase classname="tests3.test_report.ParseTestResultsTest" name="test_simple" time="0.00165796279907" rerun="1"/>
+        <testcase classname="test_simple.SampleTest" name="test_falsehood" time="0.50" rerun="3">
+            <system-out>Running SampleTest</system-out>
+            <system-out>Running SampleTest</system-out>
+        </testcase>
+    </testsuite>
+</testsuites>"""
+
 SAMPLE_XUNIT_DOUBLE_CASES = """<?xml version="1.0" encoding="utf-8"?>
 <testsuite errors="1" failures="2" name="pytest" skips="0" tests="2" time="0.019">
   <testcase classname="test_simple.SampleTest" name="test_falsehood" time="0.25" rerun="3">

tests/changes/artifacts/test_bazel_target.py

@@ -0,0 +1,104 @@
+import mock
+import pytest
+
+from cStringIO import StringIO
+
+from changes.artifacts.bazel_target import BazelTargetHandler
+from changes.constants import Result, Status
+from changes.models.bazeltarget import BazelTarget
+from changes.testutils import (
+    SAMPLE_XUNIT, SAMPLE_XUNIT_MULTIPLE_SUITES, SAMPLE_XUNIT_MULTIPLE_SUITES_COMPLETE_TIME
+)
+from changes.testutils.cases import TestCase
+
+
+@pytest.mark.parametrize('artifact_name, target_name', [
+    ('artifactstore/some/path/here/here_test/test.bazel.xml',
+     '//some/path/here:here_test'),
+    ('artifactstore/some/path/here/target/test.bazel.xml',
+     '//some/path/here:target'),
+    ('artifactstore/here_test/test.bazel.xml', '//:here_test'),
+])
+def test_get_target_name(artifact_name, target_name):
+    artifact = mock.MagicMock()
+    artifact.name = artifact_name
+    handler = BazelTargetHandler(None)
+    assert handler._get_target_name(artifact) == target_name
+
+
+class BazelTargetTestCase(TestCase):
+
+    def test_single(self):
+        project = self.create_project()
+        build = self.create_build(project)
+        job = self.create_job(build)
+        jobphase = self.create_jobphase(job)
+        jobstep = self.create_jobstep(jobphase)
+        artifact = self.create_artifact(
+            jobstep, 'artifactstore/some/target/target_test/test.bazel.xml')
+
+        handler = BazelTargetHandler(jobstep)
+
+        fp = StringIO(SAMPLE_XUNIT)
+        tests = handler.process(fp, artifact)
+
+        target = BazelTarget.query.filter(
+            BazelTarget.name == '//some/target:target_test', BazelTarget.step == jobstep).limit(1).first()
+        assert target.job == job
+        assert target.status is Status.finished
+        assert target.result is Result.failed
+        assert target.duration == 77
+        shas = [t.name_sha for t in target.tests]
+        for test_result in tests:
+            assert test_result.name_sha in shas
+        assert len(tests) == len(target.tests)
+
+    def test_multiple_duration_none(self):
+        project = self.create_project()
+        build = self.create_build(project)
+        job = self.create_job(build)
+        jobphase = self.create_jobphase(job)
+        jobstep = self.create_jobstep(jobphase)
+        artifact = self.create_artifact(
+            jobstep, 'artifactstore/some/target/target_test/test.bazel.xml')
+
+        handler = BazelTargetHandler(jobstep)
+
+        fp = StringIO(SAMPLE_XUNIT_MULTIPLE_SUITES)
+        tests = handler.process(fp, artifact)
+
+        target = BazelTarget.query.filter(
+            BazelTarget.name == '//some/target:target_test', BazelTarget.step == jobstep).limit(1).first()
+        assert target.job == job
+        assert target.status is Status.finished
+        assert target.result is Result.failed
+        assert target.duration is None
+        shas = [t.name_sha for t in target.tests]
+        for test_result in tests:
+            assert test_result.name_sha in shas
+        assert len(tests) == len(target.tests)
+
+    def test_multiple_duration_complete(self):
+        project = self.create_project()
+        build = self.create_build(project)
+        job = self.create_job(build)
+        jobphase = self.create_jobphase(job)
+        jobstep = self.create_jobstep(jobphase)
+        artifact = self.create_artifact(
+            jobstep, 'artifactstore/some/target/target_test/test.bazel.xml')
+
+        handler = BazelTargetHandler(jobstep)
+
+        fp = StringIO(SAMPLE_XUNIT_MULTIPLE_SUITES_COMPLETE_TIME)
+        tests = handler.process(fp, artifact)
+
+        target = BazelTarget.query.filter(
+            BazelTarget.name == '//some/target:target_test', BazelTarget.step == jobstep).limit(1).first()
+        assert target.job == job
+        assert target.status is Status.finished
+        assert target.result is Result.passed
+        assert target.duration == 1577
+        shas = [t.name_sha for t in target.tests]
+        for test_result in tests:
+            assert test_result.name_sha in shas
+        assert len(tests) == len(target.tests)