Merge pull request #1840 from wlach/1301729

Bug 1301729 - Store buildbot request id as a job detail

tests/e2e/test_client_job_ingestion.py

@@ -418,3 +418,35 @@ def test_post_job_with_default_tier(test_project, result_set_stored,
         job = [x for x in jobs_model.get_job_list(0, 20)
                if x['job_guid'] == job_guid][0]
         assert job['tier'] == 1
+
+
+def test_post_job_with_buildapi_artifact(test_project, result_set_stored,
+                                         mock_post_json):
+    """
+    test submitting a job with a buildapi artifact gets that stored (and
+    we update the job object)
+    """
+    tjc = client.TreeherderJobCollection()
+    job_guid = 'd22c74d4aa6d2a1dcba96d95dccbd5fdca70cf33'
+    tj = client.TreeherderJob({
+        'project': test_project,
+        'revision': result_set_stored[0]['revision'],
+        'job': {
+            'artifacts': [],
+            'job_guid': job_guid,
+            'state': 'completed',
+        }
+    })
+    tj.add_artifact("buildapi", "json",
+                    json.dumps({"buildername": "Windows 8 64-bit cheezburger",
+                                "request_id": 1234}))
+    tjc.add(tj)
+    post_collection(test_project, tjc)
+
+    assert Job.objects.count() == 1
+    assert JobDetail.objects.count() == 1
+
+    buildbot_request_id_detail = JobDetail.objects.all()[0]
+    assert buildbot_request_id_detail.title == 'buildbot_request_id'
+    assert buildbot_request_id_detail.value == str(1234)
+    assert buildbot_request_id_detail.url is None

tests/webapp/api/test_job_details_api.py

@@ -87,3 +87,11 @@ def test_job_details(test_repository, webapp):
     assert len(resp.json['results']) == 1
     assert set([v['job_guid'] for v in resp.json['results']]) == set(
         ['ijkl'])
+
+    # filter to just get those with a specific title
+    resp = webapp.get(reverse('jobdetail-list') +
+                      '?title=title')
+    print resp.json
+    assert resp.status_int == 200
+    assert len(resp.json['results']) == 1
+    assert set([v['job_guid'] for v in resp.json['results']]) == set(['abcd'])

treeherder/model/derived/artifacts.py

@@ -255,6 +255,17 @@ def load_job_artifacts(self, artifact_data):
                     self._adapt_job_artifact_collection(
                         artifact, job_artifact_list,
                         job.project_specific_id)
+                elif artifact_name == 'buildapi':
+                    buildbot_request_id = json.loads(artifact['blob']).get(
+                        'request_id')
+                    if buildbot_request_id:
+                        JobDetail.objects.update_or_create(
+                            job=job,
+                            title='buildbot_request_id',
+                            value=str(buildbot_request_id))
+                    self._adapt_job_artifact_collection(
+                        artifact, job_artifact_list,
+                        job.project_specific_id)
                 else:
                     self._adapt_job_artifact_collection(
                         artifact, job_artifact_list,

treeherder/model/management/commands/migrate_buildapi_artifacts.py

@@ -0,0 +1,101 @@
+import time
+import zlib
+from optparse import make_option
+
+import concurrent.futures
+import MySQLdb
+import simplejson as json
+from django.conf import settings
+from django.core.management.base import BaseCommand
+
+from treeherder.model.models import (Datasource,
+                                     Job,
+                                     JobDetail,
+                                     Repository)
+
+
+class Command(BaseCommand):
+
+    help = 'Migrate existing text log summary artifacts to main database'
+    option_list = BaseCommand.option_list + (
+        make_option('--project',
+                    action='append',
+                    dest='project',
+                    help='Filter operation to particular project(s)',
+                    type='string'),
+        make_option('--batch',
+                    dest='batch_size',
+                    help='Number of artifacts to migrate in interval',
+                    type='int',
+                    default=10000),
+        make_option('--interval',
+                    dest='interval',
+                    help='Wait specified interval between migrations',
+                    type='float',
+                    default=0.0))
+
+    def handle(self, *args, **options):
+        if options['project']:
+            projects = options['project']
+        else:
+            projects = Datasource.objects.values_list('project', flat=True)
+
+        for ds in Datasource.objects.filter(project__in=projects):
+            print ds.project
+            try:
+                repository = Repository.objects.get(name=ds.project)
+            except Repository.DoesNotExist:
+                self.stderr.write('No repository for datasource project {}, skipping'.format(
+                    ds.project))
+                continue
+
+            db_options = settings.DATABASES['default'].get('OPTIONS', {})
+            db = MySQLdb.connect(
+                host=settings.DATABASES['default']['HOST'],
+                db=ds.name,
+                user=settings.DATABASES['default']['USER'],
+                passwd=settings.DATABASES['default'].get('PASSWORD') or '',
+                **db_options
+            )
+            c = db.cursor()
+            offset = 0
+            limit = options['batch_size']
+
+            while True:
+                job_id_pairs = Job.objects.filter(
+                    id__gt=offset,
+                    repository=repository).values_list(
+                        'id', 'project_specific_id')[:limit]
+                if len(job_id_pairs) == 0:
+                    break
+                ds_job_ids = set([job_id_pair[1] for job_id_pair in job_id_pairs])
+                # filter out those job ids for which we already have
+                # generated job details
+                ds_job_ids -= set(JobDetail.objects.filter(
+                    title='buildbot_request_id',
+                    job__repository=repository,
+                    job__project_specific_id__in=ds_job_ids).values_list(
+                        'job__project_specific_id', flat=True))
+                start = time.time()
+                if ds_job_ids:
+                    job_id_mapping = dict((project_specific_id, job_id) for
+                                          (job_id, project_specific_id) in
+                                          job_id_pairs)
+                    c.execute("""SELECT job_id, `blob` from job_artifact where `name` = 'buildapi' and job_id in ({})""".format(",".join([str(job_id) for job_id in ds_job_ids])))
+
+                    def unwrap(row):
+                        request_id = json.loads(zlib.decompress(row[1]))['request_id']
+                        return (row[0], request_id)
+
+                    request_id_details = []
+                    with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor:
+                        for (ds_job_id, request_id) in executor.map(unwrap, c.fetchall()):
+                            request_id_details.append(JobDetail(
+                                job_id=job_id_mapping[ds_job_id],
+                                title='buildbot_request_id',
+                                value=str(request_id)))
+                        JobDetail.objects.bulk_create(request_id_details)
+                self.stdout.write('{} ({})'.format(offset, time.time() - start), ending='')
+                self.stdout.flush()
+                offset = max([job_id_pair[0] for job_id_pair in job_id_pairs])
+            print '\n'

treeherder/webapp/api/jobs.py

@@ -359,11 +359,13 @@ class NumberInFilter(django_filters.filters.BaseInFilter,
                                     lookup_expr='in')
         job_guid = django_filters.CharFilter(name='job__guid')
         job__guid = django_filters.CharFilter(name='job__guid')  # for backwards compat
+        title = django_filters.CharFilter(name='title')
         repository = django_filters.CharFilter(name='job__repository__name')
 
         class Meta:
             model = JobDetail
-            fields = ['job_guid', 'job__guid', 'job_id__in', 'repository']
+            fields = ['job_guid', 'job__guid', 'job_id__in', 'title',
+                      'repository']
 
     filter_backends = [filters.DjangoFilterBackend]
     filter_class = JobDetailFilter

ui/js/controllers/jobs.js

@@ -90,13 +90,13 @@ treeherderApp.controller('ResultSetCtrl', [
     'thUrl', 'thServiceDomain', 'thResultStatusInfo', 'thDateFormat',
     'ThResultSetStore', 'thEvents', 'thJobFilters', 'thNotify',
     'thBuildApi', 'thPinboard', 'ThResultSetModel', 'dateFilter',
-    'ThModelErrors', 'ThJobModel', 'ThJobArtifactModel',
+    'ThModelErrors', 'ThJobModel', 'ThJobDetailModel',
     function ResultSetCtrl(
         $scope, $rootScope, $http, ThLog, $location,
         thUrl, thServiceDomain, thResultStatusInfo, thDateFormat,
         ThResultSetStore, thEvents, thJobFilters, thNotify,
         thBuildApi, thPinboard, ThResultSetModel, dateFilter, ThModelErrors, ThJobModel,
-        ThJobArtifactModel) {
+        ThJobDetailModel) {
 
         $scope.getCountClass = function(resultStatus) {
             return thResultStatusInfo(resultStatus).btnClass;
@@ -200,14 +200,16 @@ treeherderApp.controller('ResultSetCtrl', [
             if (singleJobSelected) {
                 ThJobModel.cancel($scope.repoName, job.id).then(function() {
                     // XXX: Remove this after 1134929 is resolved.
-                    ThJobArtifactModel.get_list(
-                        {name: "buildapi", "type": "json", job_id: job.id}).then(
-                            function(artifactList) {
-                                if (artifactList.length) {
-                                    var requestId = artifactList[0].blob.request_id;
-                                    return thBuildApi.cancelJob($scope.repoName, requestId);
-                                }
-                            });
+                    ThJobDetailModel.getJobDetails({
+                        title: "buildbot_request_id",
+                        job_id: job.id
+                    }).then(function(data) {
+                        // non-buildbot jobs will have no request id, and that's ok (they
+                        // are covered above)
+                        if (data.length) {
+                            return thBuildApi.cancelJob($scope.repoName, data[0].value);
+                        }
+                    });
                 }).catch(function(e) {
                     thNotify.send(
                         ThModelErrors.format(e, "Unable to cancel job"),

ui/js/controllers/logviewer.js

@@ -258,7 +258,7 @@ logViewerApp.controller('LogviewerCtrl', [
                     $scope.logProperties.push({label: "Revision", value: revision});
                 });
 
-                ThJobDetailModel.getJobDetails(job.job_guid).then(function(jobDetails) {
+                ThJobDetailModel.getJobDetails({job_guid: job.job_guid}).then(function(jobDetails) {
                     $scope.job_details = jobDetails;
                 });
             }, function () {

ui/js/models/job_detail.js

@@ -3,12 +3,12 @@
 treeherder.factory('ThJobDetailModel', [
     '$http', 'thUrl', function($http, thUrl) {
         return {
-            getJobDetails: function(jobGuid, config) {
+            getJobDetails: function(params, config) {
                 config = config || {};
                 var timeout = config.timeout || null;
 
                 return $http.get(thUrl.getRootUrl("/jobdetail/"), {
-                    params: { job_guid: jobGuid },
+                    params: params,
                     timeout: timeout
                 }).then(function(response) {
                     return response.data.results;

ui/js/models/resultsets_store.js

@@ -328,7 +328,8 @@ treeherder.factory('ThResultSetStore', [
                 // This extra search is important to avoid confusion with Action Tasks
                 var decision_task = _.find(platform.groups[0].jobs, {"job_type_symbol": "D"});
                 var job_guid = decision_task.job_guid;
-                tcURLPromise = ThJobDetailModel.getJobDetails(job_guid, {timeout: null});
+                tcURLPromise = ThJobDetailModel.getJobDetails({job_guid: job_guid},
+                                                              {timeout: null});
             }
             if (!tcURLPromise) {
                 // Here we are passing false to the results instead of skipping the promise

ui/plugins/controller.js

@@ -130,12 +130,9 @@ treeherder.controller('PluginCtrl', [
                     $scope.repoName, job.id,
                     {timeout: selectJobPromise});
 
-                var buildapiArtifactPromise = ThJobArtifactModel.get_list(
-                    {name: "buildapi", "type": "json", job_id: job.id},
-                    {timeout: selectJobPromise});
-
                 var jobDetailPromise = ThJobDetailModel.getJobDetails(
-                    job.job_guid, {timeout: selectJobPromise});
+                    {job_guid: job.job_guid},
+                    {timeout: selectJobPromise});
 
                 var jobLogUrlPromise = ThJobLogUrlModel.get_list(
                     job.id,
@@ -146,7 +143,6 @@ treeherder.controller('PluginCtrl', [
 
                 return $q.all([
                     jobPromise,
-                    buildapiArtifactPromise,
                     jobDetailPromise,
                     jobLogUrlPromise,
                     phSeriesPromise
@@ -165,35 +161,32 @@ treeherder.controller('PluginCtrl', [
                     // set the tab options and selections based on the selected job
                     initializeTabs($scope.job);
 
-                    // the second result come from the buildapi artifact promise
-                    var buildapi_artifact = results[1];
-                    // if this is a buildbot job use the buildername for searching
-                    if (buildapi_artifact.length > 0 &&
-                        _.has(buildapi_artifact[0], 'blob')){
-                        // this is needed to cancel/retrigger jobs
-                        $scope.artifacts.buildapi = buildapi_artifact[0];
-                    }
-
                     // filtering values for data fields and signature
                     $scope.jobSearchStr = $scope.job.get_title();
                     $scope.jobSearchSignature = $scope.job.signature;
                     $scope.jobSearchStrHref = getJobSearchStrHref($scope.jobSearchStr);
                     $scope.jobSearchSignatureHref = getJobSearchStrHref($scope.job.signature);
 
-                    // the third result comes from the job detail promise
-                    $scope.job_details = results[2];
-                    // incorporate the buildername into the job details if it is present
-                    if ($scope.artifacts.buildapi) {
-                        $scope.artifacts.buildapi.blob.title = "Buildername";
-                        $scope.artifacts.buildapi.blob.value = $scope.artifacts.buildapi.blob.buildername;
-                        $scope.job_details = $scope.job_details.concat($scope.artifacts.buildapi.blob);
+                    // the second result comes from the job detail promise
+                    $scope.job_details = results[1];
+
+                    // incorporate the buildername into the job details if this is a buildbot job
+                    // (i.e. it has a buildbot request id)
+                    var buildbotRequestIdDetail = _.find($scope.job_details,
+                                                   {title: 'buildbot_request_id'});
+                    if (buildbotRequestIdDetail) {
+                        $scope.job_details = $scope.job_details.concat({
+                            title: "Buildername",
+                            value: $scope.job.ref_data_name
+                        });
                         $scope.buildernameIndex = _.findIndex($scope.job_details, {title: "Buildername"});
+                        $scope.job.buildbot_request_id = parseInt(buildbotRequestIdDetail.value);
                     }
 
-                    // the fourth result comes from the jobLogUrl artifact
+                    // the third result comes from the jobLogUrl artifact
                     // exclude the json log URLs
                     $scope.job_log_urls = _.reject(
-                        results[3],
+                        results[2],
                         function(log) {
                             return log.name.endsWith("_json");
                         });
@@ -217,7 +210,7 @@ treeherder.controller('PluginCtrl', [
                     }
                     $scope.resultStatusShading = "result-status-shading-" + thResultStatus($scope.job);
 
-                    var performanceData = results[4];
+                    var performanceData = results[3];
                     if (performanceData) {
                         var seriesList = [];
                         $scope.perfJobDetail = [];
@@ -326,16 +319,6 @@ treeherder.controller('PluginCtrl', [
                    ($scope.job.state === "pending" || $scope.job.state === "running");
         };
 
-        /**
-         * Get the build_id needed to cancel or retrigger from the currently
-         * selected job.
-         */
-        var getBuildbotRequestId = function() {
-            if ($scope.artifacts.buildapi) {
-                return $scope.artifacts.buildapi.blob.request_id;
-            }
-        };
-
         $scope.retriggerJob = function(jobs) {
             if ($scope.user.loggedin) {
                 var job_id_list = _.pluck(jobs, 'id');
@@ -346,14 +329,12 @@ treeherder.controller('PluginCtrl', [
                 // to the self serve api (which does not listen over pulse!).
                 ThJobModel.retrigger($scope.repoName, job_id_list).then(function() {
                     // XXX: Remove this after 1134929 is resolved.
-                    return ThJobArtifactModel.get_list({"name": "buildapi",
-                                                        "type": "json",
-                                                        "count": job_id_list.length,
-                                                        "job_id__in": job_id_list.join(',')})
+                    return ThJobDetailModel.getJobDetails({"title": "buildbot_request_id",
+                                                           "job_id__in": job_id_list.join(',')})
                         .then(function(data) {
-                            var request_id_list = _.pluck(_.pluck(data, 'blob'), 'request_id');
-                            _.each(request_id_list, function(request_id) {
-                                thBuildApi.retriggerJob($scope.repoName, request_id);
+                            var requestIdList = _.pluck(data, 'value');
+                            requestIdList.forEach(function(requestId) {
+                                thBuildApi.retriggerJob($scope.repoName, requestId);
                             });
                         });
                 }).then(function() {
@@ -404,7 +385,7 @@ treeherder.controller('PluginCtrl', [
                 // See note in retrigger logic.
                 ThJobModel.cancel($scope.repoName, $scope.job.id).then(function() {
                   // XXX: Remove this after 1134929 is resolved.
-                    var requestId = getBuildbotRequestId();
+                    var requestId = $scope.job.buildbot_request_id;
                     if (requestId) {
                         return thBuildApi.cancelJob($scope.repoName, requestId);
                     }