Artifacts collection for bazel projects

Summary: This updates the build step to properly specify the config BAZEL_TEST_OUTPUT_RELATIVE_PATH as the path to search artifacts in, and to collect anything under that directly matching `*.xml`.

Test Plan: unit tests

Reviewers: anupc

Reviewed By: anupc

Subscribers: changesbot, kylec

Differential Revision: https://tails.corp.dropbox.com/D222677

changes/buildsteps/default.py

@@ -70,7 +70,7 @@ def __init__(self, commands=None, repo_path=None, path=None, env=None,
                  artifacts=DEFAULT_ARTIFACTS, release=DEFAULT_RELEASE,
                  max_executors=10, cpus=4, memory=8 * 1024, clean=True,
                  debug_config=None, test_stats_from=None, cluster=None,
-                 other_repos=None,
+                 other_repos=None, artifact_search_path=None,
                  **kwargs):
         """
         Constructor for DefaultBuildStep.
@@ -126,6 +126,9 @@ def __init__(self, commands=None, repo_path=None, path=None, env=None,
                 can be specified with "backend": "hg". Default revision is
                 "origin/master" or "default" (for hg), but an explicit revision
                 can be specified with "revision".
+            artifact_search_path: The path in which test artifacts can be
+                found in, relative to the `repo_path`. This defaults to the
+                value for `path`.
         """
         if commands is None:
             raise ValueError("Missing required config: need commands")
@@ -145,6 +148,7 @@ def __init__(self, commands=None, repo_path=None, path=None, env=None,
             # default repo_path to path if none specified
             self.repo_path = path or DEFAULT_PATH
             self.path = self.repo_path
+        self.artifact_search_path = os.path.join(self.repo_path, artifact_search_path) if artifact_search_path else self.path
         self.release = release
         self.max_executors = max_executors
         self.resources = {
@@ -480,7 +484,7 @@ def _image_for_job_id(self, job_id):
 
     def get_allocation_params(self, jobstep):
         params = {
-            'artifact-search-path': self.path,
+            'artifact-search-path': self.artifact_search_path,
             'artifacts-server': current_app.config['ARTIFACTS_SERVER'],
             'adapter': self.get_client_adapter(),
             'server': build_internal_uri('/api/0/'),

changes/config.py

@@ -95,6 +95,8 @@ def create_app(_read_config=True, **config):
     app.config['HTTP_PORT'] = 5000
     app.config['SEND_FILE_MAX_AGE_DEFAULT'] = 0
 
+    app.config['BAZEL_TEST_OUTPUT_RELATIVE_PATH'] = 'bazel-testlogs/'
+
     app.config['API_TRACEBACKS'] = True
 
     # Expiration delay between when a snapshot image becomes superceded and when

changes/models/jobplan.py

@@ -204,6 +204,8 @@ def get_build_step_for_job(cls, job_id):
                     {'script': sync_encap_pkgs(project_config), 'type': 'setup'},
                     {'script': collect_bazel_targets(project_config['bazel.targets']), 'type': 'collect_tests'},
                 ],
+                artifacts=['*.xml'],
+                artifact_search_path=current_app.config['BAZEL_TEST_OUTPUT_RELATIVE_PATH'],
             )
             return jobplan, implementation
 

tests/changes/buildsteps/test_default.py

@@ -565,6 +565,29 @@ def test_get_allocation_params(self):
             'dist': 'ubuntu',
         }
 
+    def test_get_allocation_params_with_artifact_search_path(self):
+        project = self.create_project()
+        build = self.create_build(project)
+        job = self.create_job(build)
+        jobphase = self.create_jobphase(job)
+        jobstep = self.create_jobstep(jobphase)
+
+        buildstep = self.get_buildstep(repo_path='source', path='tests', artifact_search_path='out')
+        result = buildstep.get_allocation_params(jobstep)
+        assert result == {
+            'adapter': 'basic',
+            'server': 'http://changes-int.example.com/api/0/',
+            'jobstep_id': jobstep.id.hex,
+            'release': 'precise',
+            's3-bucket': 'snapshot-bucket',
+            'pre-launch': 'echo pre',
+            'post-launch': 'echo post',
+            'artifacts-server': 'http://localhost:1234',
+            'artifact-search-path': 'source/out',
+            'use-external-env': 'false',
+            'dist': 'ubuntu',
+        }
+
     def test_get_allocation_params_for_snapshotting(self):
         project = self.create_project()
         build = self.create_build(project)

tests/changes/models/test_job.py

@@ -1,9 +1,11 @@
 from __future__ import absolute_import
 
 import mock
+import os
 
 from flask import current_app
 
+from changes.buildsteps.default import DEFAULT_PATH
 from changes.models.command import CommandType
 from changes.models.jobplan import JobPlan
 from changes.testutils import TestCase
@@ -92,3 +94,6 @@ def test_autogenerated_commands(self, get_config):
 
         assert implementation.commands[2].type == CommandType.collect_tests
         assert implementation.commands[2].script == collect_tests_expected
+
+        assert implementation.artifact_search_path == os.path.join(DEFAULT_PATH, current_app.config['BAZEL_TEST_OUTPUT_RELATIVE_PATH'])
+        assert implementation.artifacts == ['*.xml']