Bug 1270629 - Add initial parts of jobdetail model/API

Ingestion not turned on (yet), pending initial migration of data

tests/webapp/api/test_job_details_api.py

@@ -0,0 +1,54 @@
+from django.core.urlresolvers import reverse
+
+from treeherder.model.models import (Job,
+                                     JobDetail)
+
+
+def test_job_details(test_repository, webapp):
+    details = {
+        'abcd': {
+            'title': 'title',
+            'value': 'value1',
+            'url': None
+        },
+        'efgh': {
+            'title': None,
+            'value': 'value2',
+            'url': None
+        },
+        'ijkl': {
+            'title': 'title3',
+            'value': 'value3',
+            'url': 'https://localhost/foo'
+        }
+    }
+
+    # create some job details for some fake jobs
+    i = 0
+    for (job_guid, params) in details.iteritems():
+        print job_guid
+        job = Job.objects.create(guid=job_guid,
+                                 repository=test_repository,
+                                 project_specific_id=i)
+        JobDetail.objects.create(
+            job=job, **params)
+        i += 1
+    print JobDetail.objects.filter(job__guid='abcd')
+    # get them all
+    resp = webapp.get(reverse('jobdetail-list'))
+    assert resp.status_int == 200
+    assert len(resp.json['results']) == 3
+    for result in resp.json['results']:
+        job_guid = result['job_guid']
+        del result['job_guid']
+        assert result == details[job_guid]
+
+    # filter to just get one guid at a time
+    for (guid, detail) in details.iteritems():
+        resp = webapp.get(reverse('jobdetail-list') + '?job__guid={}'.format(
+            guid))
+        assert resp.status_int == 200
+        assert len(resp.json['results']) == 1
+        result = resp.json['results'][0]
+        del result['job_guid']
+        assert result == details[guid]

treeherder/model/management/commands/migrate_job_info_artifacts.py

@@ -0,0 +1,78 @@
+import sys
+import time
+from optparse import make_option
+
+from django.core.management.base import BaseCommand
+from django.db import transaction
+
+from treeherder.model.derived import ArtifactsModel
+from treeherder.model.models import (Datasource,
+                                     Job,
+                                     JobDetail,
+                                     Repository)
+
+
+class Command(BaseCommand):
+
+    help = 'Migrate existing jobs to intermediate jobs table'
+    option_list = BaseCommand.option_list + (
+        make_option('--project',
+                    action='append',
+                    dest='project',
+                    help='Filter deletion to particular project(s)',
+                    type='string'),
+        make_option('--interval',
+                    dest='interval',
+                    help='Wait specified number of seconds between job info migrations',
+                    type='float',
+                    default=0.0))
+
+    def handle(self, *args, **options):
+        if options['project']:
+            datasources = Datasource.objects.filter(
+                project__in=options['project'])
+        else:
+            datasources = Datasource.objects.all()
+        for ds in datasources:
+            self.stdout.write('{}\n'.format(ds.project))
+
+            repository = Repository.objects.get(name=ds.project)
+            offset = 0
+            limit = 10000
+            while True:
+                job_ids = set(Job.objects.order_by(
+                    'project_specific_id').filter(
+                        project_specific_id__gt=offset,
+                        repository=repository).values_list(
+                            'project_specific_id', flat=True)[:limit])
+                if len(job_ids) == 0:
+                    break
+                max_job_id = max(job_ids)
+                # filter out those job ids for which we already have
+                # generated job details
+                job_ids -= set(JobDetail.objects.filter(
+                    job__repository=repository,
+                    job__project_specific_id__in=job_ids).values_list(
+                        'job__project_specific_id', flat=True))
+                if job_ids:
+                    with ArtifactsModel(ds.project) as am:
+                        am.DEBUG = False
+                        artifacts = am.get_job_artifact_list(0, 10000, {
+                            'job_id': set([('IN', tuple(job_ids))]),
+                            'name': set([("=", "Job Info")])})
+                        with transaction.atomic():
+                            for artifact in artifacts:
+                                for job_detail_dict in artifact['blob']['job_details']:
+                                    intermediate_job = Job.objects.get(
+                                        repository=repository,
+                                        project_specific_id=artifact['job_id'])
+                                    JobDetail.objects.create(
+                                        job=intermediate_job,
+                                        title=job_detail_dict.get('title'),
+                                        value=job_detail_dict['value'],
+                                        url=job_detail_dict.get('url'))
+                self.stdout.write('{} '.format(offset))
+                sys.stdout.flush()
+                offset = max_job_id
+                time.sleep(options['interval'])
+        self.stdout.write("\n")

treeherder/model/migrations/0021_add_jobdetail_model.py

@@ -0,0 +1,28 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+import treeherder.model.fields
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('model', '0020_update_job_name_length'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='JobDetail',
+            fields=[
+                ('id', treeherder.model.fields.BigAutoField(serialize=False, primary_key=True)),
+                ('title', models.CharField(max_length=200, null=True)),
+                ('value', models.CharField(max_length=200)),
+                ('url', models.URLField(null=True)),
+                ('job', treeherder.model.fields.FlexibleForeignKey(to='model.Job')),
+            ],
+            options={
+                'db_table': 'job_detail',
+            },
+        ),
+    ]

treeherder/model/models.py

@@ -557,6 +557,30 @@ def __str__(self):
                                         self.project_specific_id)
 
 
+class JobDetail(models.Model):
+    '''
+    Represents metadata associated with a job
+
+    There can be (and usually is) more than one of these associated with
+    each job
+    '''
+    id = BigAutoField(primary_key=True)
+    job = FlexibleForeignKey(Job)
+    title = models.CharField(max_length=200, null=True)
+    value = models.CharField(max_length=200)
+    url = models.URLField(null=True)
+
+    class Meta:
+        db_table = "job_detail"
+
+    def __str__(self):
+        return "{0} {1} {2} {3} {4}".format(self.id,
+                                            self.job.guid,
+                                            self.title,
+                                            self.value,
+                                            self.url)
+
+
 class FailureLineManager(models.Manager):
     def unmatched_for_job(self, repository, job_guid):
         return FailureLine.objects.filter(

treeherder/webapp/api/jobs.py

@@ -1,4 +1,6 @@
-from rest_framework import viewsets
+from rest_framework import (filters,
+                            pagination,
+                            viewsets)
 from rest_framework.decorators import (detail_route,
                                        list_route)
 from rest_framework.permissions import IsAuthenticated
@@ -7,6 +9,7 @@
 
 from treeherder.model.derived import ArtifactsModel
 from treeherder.model.models import (FailureLine,
+                                     JobDetail,
                                      OptionCollection,
                                      TextLogSummary)
 from treeherder.webapp.api import (permissions,
@@ -294,3 +297,21 @@ def create(self, request, project, jm):
         jm.store_job_data(request.data)
 
         return Response({'message': 'Job successfully updated'})
+
+
+class JobDetailViewSet(viewsets.ReadOnlyModelViewSet):
+    '''
+    Endpoint for retrieving metadata (e.g. links to artifacts, file sizes)
+    associated with a particular job
+    '''
+    queryset = JobDetail.objects.all().select_related('job__guid')
+    serializer_class = serializers.JobDetailSerializer
+
+    filter_backends = (filters.DjangoFilterBackend, filters.OrderingFilter)
+    filter_fields = ['job__guid']
+
+    class JobDetailPagination(pagination.LimitOffsetPagination):
+        default_limit = 100
+        max_limit = 100
+
+    pagination_class = JobDetailPagination

treeherder/webapp/api/serializers.py

@@ -167,3 +167,14 @@ class TextLogSummarySerializer(serializers.ModelSerializer):
 
     class Meta:
         model = models.TextLogSummary
+
+
+class JobDetailSerializer(serializers.ModelSerializer):
+
+    job_guid = serializers.SlugRelatedField(
+        slug_field="guid", source="job",
+        queryset=models.Job.objects.all())
+
+    class Meta:
+        model = models.JobDetail
+        fields = ['job_guid', 'title', 'value', 'url']

treeherder/webapp/api/urls.py

@@ -133,6 +133,8 @@ class ExtendedRouter(routers.DefaultRouter):
                         base_name='performance-frameworks')
 default_router.register(r'bugzilla', bugzilla.BugzillaViewSet,
                         base_name='bugzilla')
+default_router.register(r'jobdetail', jobs.JobDetailViewSet,
+                        base_name='jobdetail')
 
 urlpatterns = [
     url(r'^project/(?P<project>[\w-]{0,50})/',